def check_vim_capacity(self, vim_request):
LOG.debug("Invoking check_vim_capacity api")
path = '/{}/{}'.format(self.version, 'check_vim_capacity')
data = {}
data['vCPU'] = vim_request['vCPU']
data['Memory'] = vim_request['Memory']['quantity']
data['Storage'] = vim_request['Storage']['quantity']
data['VIMs'] = vim_request['VIMs']
response = self._request('post',
path=path,
data=data,
context="vim capacity",
value="all")
LOG.debug("Response check_vim_capacity api - {}".format(response))
if response is None or response.status_code != 200:
return None
body = response.json()
if body:
vims = body.get("VIMs")
if vims is None:
LOG.error(
_LE("Unable to get VIMs with cpu-{}, memory-{}, disk-{}").
format(data['vCPU'], data['Memory'], data['Storage']))
return vims
else:
LOG.error(
_LE("Unable to get VIMs from Multicloud with "
"requirement {}").format(data))
return None
def _request(self, method='get', path='/', data=None,
context=None, value=None):
"""Performs HTTP request."""
headers = {
'X-FromAppId': 'CONDUCTOR',
'X-TransactionId': str(uuid.uuid4()),
}
kwargs = {
"method": method,
"path": path,
"headers": headers,
"data": data,
}
start_time = time.time()
response = self.rest.request(**kwargs)
elapsed = time.time() - start_time
LOG.debug("Total time for Multicloud request "
"({0:}: {1:}): {2:.3f} sec".format(context, value, elapsed))
if response is None:
LOG.error(_LE("No response from Multicloud ({}: {})").
format(context, value))
elif response.status_code != 200:
LOG.error(_LE("Multicloud request ({}: {}) returned HTTP "
"status {} {}, link: {}{}").
format(context, value,
response.status_code, response.reason,
self.base, path))
return response
def remote_api(aafUser):
server_url = CONF.aaf_api.aaf_url+aafUser
kwargs = {
"server_url": server_url,
"retries": CONF.aaf_api.aaf_retries,
"username": CONF.aaf_api.username,
"password": CONF.aaf_api.password,
"log_debug": LOG.debug,
"read_timeout": CONF.aaf_api.aaf_timeout,
"cert_file": CONF.aaf_api.aaf_cert_file,
"cert_key_file": CONF.aaf_api.aaf_cert_key_file,
"ca_bundle_file": CONF.aaf_api.aaf_ca_bundle_file,
}
restReq = rest.REST(**kwargs)
headers = {"Accept": "application/Perms+json;q=1.0;charset=utf-8;version=2.1,application/json;q=1.0;version=2.1,*/*;q=1.0"}
rkwargs = {
"method": 'GET',
"path": '',
"headers": headers,
}
response = restReq.request(**rkwargs)
if response is None:
LOG.error(_LE("No response from AAF "))
elif response.status_code != 200:
LOG.error(_LE("AAF request returned HTTP "
"status {} {}, link: {}").
format(response.status_code, response.reason,
server_url))
return response.content
def _request(self,
method='get',
path='/',
data=None,
context=None,
value=None):
"""Performs HTTP request."""
kwargs = {
"method": method,
"path": path,
"data": data,
}
# TODO(jdandrea): Move timing/response logging into the rest helper?
start_time = time.time()
response = self.rest.request(**kwargs)
elapsed = time.time() - start_time
LOG.debug("Total time for SDN-C request "
"({0:}: {1:}): {2:.3f} sec".format(context, value, elapsed))
if response is None:
LOG.error(
_LE("No response from SDN-C ({}: {})").format(context, value))
raise Exception('SDN-C query {} timed out'.format(path))
elif response.status_code != 200:
LOG.error(
_LE("SDN-C request ({}: {}) returned HTTP "
"status {} {}, link: {}{}").format(context, value,
response.status_code,
response.reason,
self.base, path))
return response
def resolve_location(self, ctx, arg):
log_util.setLoggerFilter(LOG, ctx.get('keyspace'), ctx.get('plan_id'))
error = False
resolved_location = None
host_name = arg.get('host_name')
clli_code = arg.get('clli_code')
if host_name:
results = self.ip_ext_manager.map_method('resolve_host_location',
host_name)
elif clli_code:
results = self.ip_ext_manager.map_method('resolve_clli_location',
clli_code)
else:
results = None
# unknown location response
LOG.error(
_LE("Unknown location type from the input template."
"Expected location types are host_name"
" or clli_code."))
if results and len(results) > 0:
resolved_location = results[0]
else:
error = True
return {
'response': {
'resolved_location': resolved_location
},
'error': error
}
def call_reservation_operation(self, ctx, arg):
result = True
reserved_candidates = None
method = arg["method"]
candidate_list = arg["candidate_list"]
reservation_name = arg["reservation_name"]
reservation_type = arg["reservation_type"]
controller = arg["controller"]
request = arg["request"]
if controller == "SDN-C":
results = self.sc_ext_manager.map_method(
'call_reservation_operation',
method=method,
candidate_list=candidate_list,
reservation_name=reservation_name,
reservation_type=reservation_type,
request=request)
if results and len(results) > 0:
reserved_candidates = results[0]
else:
LOG.error(_LE("Unknown service controller: {}").format(controller))
if reserved_candidates is None or not reserved_candidates:
result = False
LOG.debug(
_LW("Unable to {} for "
"candidate {}.").format(method, reserved_candidates))
return {'response': result, 'error': not result}
else:
LOG.debug("{} for the candidate: "
"{}".format(method, reserved_candidates))
return {'response': result, 'error': not result}
def rollback_reservation(self, reservation_list):
"""Function to rollback(release) reservations"""
# TODO(snarayanan): Need to test this once the API is ready
for reservation in reservation_list:
candidate_list = reservation['candidate_list']
reservation_name = reservation['reservation_name']
reservation_type = reservation['reservation_type']
controller = reservation['controller']
request = reservation['request']
is_success = self.try_reservation_call(
method="release",
candidate_list=candidate_list,
reservation_name=reservation_name,
reservation_type=reservation_type,
controller=controller,
request=request)
if not is_success:
# rollback failed report error to SDNC
message = _LE("Unable to release reservation "
"{}").format(reservation)
LOG.error(message)
return False
# move to the next reserved candidate
return True
def encode(user_id, password):
""" Provide the basic authencation encoded value in an 'Authorization' Header """
user_pass = user_id + ':' + password
base64_val = base64.b64encode(user_pass)
authorization_val = _LE("Basic {}".format(base64_val))
return authorization_val
def get_inventory_group_candidates(self, ctx, arg):
candidate_list = arg["candidate_list"]
resolved_candidate = arg["resolved_candidate"]
candidate_names = []
error = False
service_description = 'DHV_VVIG_PAIR'
results = self.ip_ext_manager.map_method(
'get_inventory_group_pairs',
service_description=service_description)
if not results or len(results) < 1:
LOG.error(
_LE("Empty inventory group response for service: {}").format(
service_description))
error = True
else:
pairs = results[0]
if not pairs or len(pairs) < 1:
LOG.error(
_LE("No inventory group candidates found for service: {}, "
"inventory provider: {}").format(
service_description,
self.ip_ext_manager.names()[0]))
error = True
else:
LOG.debug("Inventory group pairs: {}, service: {}, "
"inventory provider: {}".format(
pairs, service_description,
self.ip_ext_manager.names()[0]))
for pair in pairs:
if resolved_candidate.get("candidate_id") == pair[0]:
candidate_names.append(pair[1])
elif resolved_candidate.get("candidate_id") == pair[1]:
candidate_names.append(pair[0])
candidate_list = [
c for c in candidate_list if c["candidate_id"] in candidate_names
]
LOG.info(
_LI("Inventory group candidates: {}, service: {}, "
"inventory provider: {}").format(
candidate_list, service_description,
self.ip_ext_manager.names()[0]))
return {'response': candidate_list, 'error': error}
def encode(user_id, password):
""" Provide the basic authencation encoded value in an 'Authorization' Header """
user_pass = user_id + ":" + password
# Here user_pass is str type but in python 3.x version, base64.b64encode is expecting byte
# like object so we need to convert the str into bytes
user_pass = user_pass.encode() # converting str into bytes form
base64_val = base64.b64encode(user_pass).decode()
authorization_val = _LE("Basic {}".format(base64_val))
return authorization_val
def _request(self,
method='get',
path='/',
data=None,
context=None,
value=None):
"""Performs HTTP request."""
headers = {
'X-FromAppId': 'AAI',
'X-TransactionId': str(uuid.uuid4()),
'X-ECOMP-InstanceID': 'AAI',
}
kwargs = {
"method": method,
"path": path,
"headers": headers,
"data": data,
"content_type": "application/octet-stream"
}
# TODO(jdandrea): Move timing/response logging into the rest helper?
start_time = time.time()
response = self.rest.request(**kwargs)
elapsed = time.time() - start_time
LOG.debug("Total time for SDC request "
"({0:}: {1:}): {2:.3f} sec".format(context, value, elapsed))
if response is None:
LOG.error(
_LE("No response from SDC ({}: {})").format(context, value))
elif response.status_code != 200:
LOG.error(
_LE("SDC request ({}: {}) returned HTTP "
"status {} {}, link: {}{}").format(context, value,
response.status_code,
response.reason,
self.base, path))
return response
def translate(self, plan):
"""Translate the plan to a format the solver can use"""
# Update the translation field and set status to TRANSLATED.
try:
LOG.info(_LI("Requesting plan {} translation").format(plan.id))
template_version = plan.template.get("homing_template_version")
if template_version in VERSIONS['BASE']:
trns = Translator(self.conf, plan.name, plan.id, plan.template)
elif template_version in VERSIONS['GENERIC']:
trns = GenericObjectiveTranslator(self.conf, plan.name,
plan.id, plan.template,
self.opt_schema)
else:
raise TranslatorException(
"conductor_template_version must be one "
"of: {}".format(', '.join(
[x for v in VERSIONS.values() for x in v])))
trns.translate()
if trns.ok:
plan.translation = trns.translation
plan.status = self.Plan.TRANSLATED
LOG.info(
_LI("Plan {} translated. Ready for solving").format(
plan.id))
LOG.info(_LI("Plan name: {}").format(plan.name))
else:
plan.message = trns.error_message
plan.status = self.Plan.ERROR
LOG.error(
_LE("Plan {} translation error encountered").format(
plan.id))
except Exception as ex:
template = "An exception of type {0} occurred, arguments:\n{1!r}"
plan.message = template.format(type(ex).__name__, ex.args)
plan.status = self.Plan.ERROR
_is_success = 'FAILURE'
while 'FAILURE' in _is_success and (self.current_time_seconds() - self.millisec_to_sec(plan.updated)) \
<= self.conf.messaging_server.timeout:
_is_success = plan.update(
condition=self.translation_owner_condition)
LOG.info(
_LI("Changing the template status from translating to {}, "
"atomic update response from MUSIC {}").format(
plan.status, _is_success))
def lock_create(self, keyspace, table, pk_value):
"""Create and acquire a lock. Returns a lock name."""
# Generate the lock name, then create/acquire the lock id.
lock_name = self._lock_name_generate(keyspace, table, pk_value)
if CONF.music_api.debug:
LOG.debug("Creating lock {}".format(lock_name))
lock_id = self._lock_id_create(lock_name)
time_now = time.time()
while not self._lock_id_acquire(lock_id):
if time.time() - time_now > self.lock_timeout:
raise IndexError(
_LE('Lock id acquire timeout: %s') % lock_name)
# Cache the lock name/id.
self.lock_ids[lock_name] = lock_id
return lock_name
def get_candidates_from_service(self, ctx, arg):
candidate_list = arg["candidate_list"]
constraint_name = arg["constraint_name"]
constraint_type = arg["constraint_type"]
controller = arg["controller"]
request = arg["request"]
request_type = arg["request_type"]
error = False
filtered_candidates = []
# call service and fetch candidates
# TODO(jdandrea): Get rid of the SDN-C reference (outside of plugin!)
if controller == "SDN-C":
service_model = request.get("service_model")
results = self.sc_ext_manager.map_method(
'filter_candidates',
request=request,
candidate_list=candidate_list,
constraint_name=constraint_name,
constraint_type=constraint_type,
request_type=request_type)
if results and len(results) > 0:
filtered_candidates = results[0]
else:
LOG.warn(
_LW("No candidates returned by service "
"controller: {}; may be a new service "
"instantiation.").format(controller))
else:
LOG.error(_LE("Unknown service controller: {}").format(controller))
# if response from service controller is empty
if filtered_candidates is None:
if service_model == "ADIOD":
LOG.error("No capacity found from SDN-GC for candidates: "
"{}".format(candidate_list))
return {'response': [], 'error': error}
else:
LOG.debug("Filtered candidates: {}".format(filtered_candidates))
candidate_list = [
c for c in candidate_list if c in filtered_candidates
]
return {'response': candidate_list, 'error': error}
def get_candidate_zone(self, ctx, arg):
candidate = arg["candidate"]
category = arg["category"]
zone = None
error = False
if category == 'region':
zone = candidate['location_id']
elif category == 'complex':
zone = candidate['complex_name']
elif category == 'country':
zone = candidate['country']
else:
error = True
if error:
LOG.error(_LE("Unresolvable zone category {}").format(category))
else:
LOG.info(_LI("Candidate zone is {}").format(zone))
return {'response': zone, 'error': error}
def solve(self, _decision_path, _candidate_list, _request):
select_list = list()
candidates_to_check = list()
demand_name = _decision_path.current_demand.name
# service-check candidates of the same inventory type
# select candidate of all other types
for candidate in _candidate_list:
if self.inventory_type == "cloud":
if candidate["inventory_type"] == "cloud":
candidates_to_check.append(candidate)
else:
select_list.append(candidate)
elif self.inventory_type == "service":
if candidate["inventory_type"] == "service":
candidates_to_check.append(candidate)
else:
select_list.append(candidate)
elif self.inventory_type == "vfmodule":
if candidate["inventory_type"] == "vfmodule":
candidates_to_check.append(candidate)
else:
select_list.append(candidate)
# call conductor data with request parameters
if len(candidates_to_check) > 0:
cei = _request.cei
request_type = _request.request_type
filtered_list = cei.get_candidates_from_service(
self.name, self.constraint_type, candidates_to_check,
self.controller, self.inventory_type, self.request,
self.cost, demand_name, request_type)
for c in filtered_list:
select_list.append(c)
else:
LOG.error(_LE("Constraint {} ({}) has no candidates of "
"inventory type {} for demand {}").format(
self.name, self.constraint_type,
self.inventory_type, demand_name)
)
_candidate_list[:] = [c for c in _candidate_list if c in select_list]
return _candidate_list
def translate(self, plan):
"""Translate the plan to a format the solver can use"""
# Update the translation field and set status to TRANSLATED.
try:
LOG.info(_LI("Requesting plan {} translation").format(plan.id))
trns = translator.Translator(self.conf, plan.name, plan.id,
plan.template)
trns.translate()
if trns.ok:
plan.translation = trns.translation
plan.status = self.Plan.TRANSLATED
LOG.info(
_LI("Plan {} translated. Ready for solving").format(
plan.id))
LOG.info(_LI("Plan name: {}").format(plan.name))
else:
plan.message = trns.error_message
plan.status = self.Plan.ERROR
LOG.error(
_LE("Plan {} translation error encountered").format(
plan.id))
except Exception as ex:
template = "An exception of type {0} occurred, arguments:\n{1!r}"
plan.message = template.format(type(ex).__name__, ex.args)
plan.status = self.Plan.ERROR
_is_success = 'FAILURE'
while 'FAILURE' in _is_success and (
self.current_time_seconds() - self.millisec_to_sec(
plan.updated)) <= self.conf.messaging_server.timeout:
_is_success = plan.update(
condition=self.translation_owner_condition)
LOG.info(
_LI("Changing the template status from translating to {}, "
"atomic update response from MUSIC {}").format(
plan.status, _is_success))
def solve(self, _decision_path, _candidate_list, _request):
'''
Solver for HPA constraint type.
:param _decision_path: decision tree
:param _candidate_list: List of candidates
:param _request: solver request
:return: candidate_list with hpa features and flavor label mapping
'''
# call conductor engine with request parameters
cei = _request.cei
demand_name = _decision_path.current_demand.name
LOG.info(_LI("Solving constraint type '{}' for demand - [{}]").format(
self.constraint_type, demand_name))
vm_label_list = self.properties.get('evaluate')
for vm_demand in vm_label_list:
id = vm_demand['id']
type = vm_demand['type']
directives = vm_demand['directives']
flavorProperties = vm_demand['flavorProperties']
response = (cei.get_candidates_with_hpa(id,
type,
directives,
_candidate_list,
flavorProperties))
_candidate_list = response
if not response:
LOG.error(_LE("No matching candidates for HPA exists").format(
id))
# Metrics to Prometheus
PC.HPA_CLOUD_REGION_UNSUCCESSFUL.labels('ONAP', 'N/A',
'ALL').inc()
break
# No need to continue.
return _candidate_list
def test_basic_auth_util(self):
self.authorization_val = _LE(
BaseAuthUtil.encode(self.userId, self.userId))
self.assertEqual('Basic UGFzc3dvcmQ6UGFzc3dvcmQ=',
self.authorization_val)
def get_candidates_with_hpa(self, ctx, arg):
'''
RPC for getting candidates flavor mapping for matching hpa
:param ctx: context
:param arg: contains input passed from client side for RPC call
:return: response candidate_list with matching label to flavor mapping
'''
error = False
candidate_list = arg["candidate_list"]
id = arg["id"]
type = arg["type"]
directives = arg["directives"]
attr = directives[0].get("attributes")
label_name = attr[0].get("attribute_name")
flavorProperties = arg["flavorProperties"]
discard_set = set()
for i in range(len(candidate_list)):
# perform this check only for cloud candidates
if candidate_list[i]["inventory_type"] != "cloud":
continue
# Check if flavor mapping for current label_name already
# exists. This is an invalid condition.
if candidate_list[i].get(
"directives") and attr[0].get("attribute_value") != "":
LOG.error(
_LE("Flavor mapping for label name {} already"
"exists").format(label_name))
continue
# RPC call to inventory provider for matching hpa capabilities
results = self.ip_ext_manager.map_method(
'match_hpa',
candidate=candidate_list[i],
features=flavorProperties)
flavor_name = None
if results and len(results) > 0 and results[0] is not None:
LOG.debug("Find results {} and results length {}".format(
results, len(results)))
flavor_info = results[0].get("flavor_map")
req_directives = results[0].get("directives")
LOG.debug("Get directives {}".format(req_directives))
else:
flavor_info = None
LOG.info(
_LW("No flavor mapping returned by "
"inventory provider: {} for candidate: {}").format(
self.ip_ext_manager.names()[0],
candidate_list[i].get("candidate_id")))
# Metrics to Prometheus
m_vim_id = candidate_list[i].get("vim-id")
if not flavor_info:
discard_set.add(candidate_list[i].get("candidate_id"))
PC.HPA_CLOUD_REGION_UNSUCCESSFUL.labels(
'ONAP', 'N/A', m_vim_id).inc()
else:
if not flavor_info.get("flavor-name"):
discard_set.add(candidate_list[i].get("candidate_id"))
PC.HPA_CLOUD_REGION_UNSUCCESSFUL.labels(
'ONAP', 'N/A', m_vim_id).inc()
else:
if not candidate_list[i].get("flavor_map"):
candidate_list[i]["flavor_map"] = {}
# Create flavor mapping for label_name to flavor
flavor_name = flavor_info.get("flavor-name")
flavor_id = flavor_info.get("flavor-id")
candidate_list[i]["flavor_map"][label_name] = flavor_name
candidate_list[i]["flavor_map"]["flavorId"] = flavor_id
# Create directives if not exist already
if not candidate_list[i].get("all_directives"):
candidate_list[i]["all_directives"] = {}
candidate_list[i]["all_directives"]["directives"] = []
# Create flavor mapping and merge directives
self.merge_directives(candidate_list, i, id, type,
directives, req_directives)
if not candidate_list[i].get("hpa_score"):
candidate_list[i]["hpa_score"] = 0
candidate_list[i]["hpa_score"] += flavor_info.get("score")
# Metrics to Prometheus
PC.HPA_CLOUD_REGION_SUCCESSFUL.labels(
'ONAP', 'N/A', m_vim_id).inc()
# return candidates not in discard set
candidate_list[:] = [
c for c in candidate_list if c['candidate_id'] not in discard_set
]
LOG.info(
_LI("Candidates with matching hpa capabilities: {}, "
"inventory provider: {}").format(
candidate_list,
self.ip_ext_manager.names()[0]))
return {'response': candidate_list, 'error': error}
def get_candidates_by_attributes(self, ctx, arg):
candidate_list = arg["candidate_list"]
# demand_name = arg["demand_name"]
properties = arg["properties"]
discard_set = set()
attributes_to_evaluate = properties.get('evaluate')
for attrib, value in attributes_to_evaluate.items():
if value == '':
continue
if attrib == 'network_roles':
role_candidates = dict()
role_list = []
nrc_dict = value
role_condition = ''
if nrc_dict:
if "all" in nrc_dict:
role_list = nrc_dict.get("all")
role_condition = "all"
elif "any" in nrc_dict:
role_list = nrc_dict.get("any")
role_condition = "any"
# if the role_list is empty do nothing
if not role_list or role_list == '':
LOG.error(
_LE("No roles available, "
"inventory provider: {}").format(
self.ip_ext_manager.names()[0]))
continue
for role in role_list:
# query inventory provider to check if
# the candidate is in role
results = self.ip_ext_manager.map_method(
'check_network_roles', network_role_id=role)
if not results or len(results) < 1:
LOG.error(
_LE("Empty response from inventory "
"provider {} for network role {}").format(
self.ip_ext_manager.names()[0], role))
continue
region_ids = results[0]
if not region_ids:
LOG.error(
_LE("No candidates from inventory provider {} "
"for network role {}").format(
self.ip_ext_manager.names()[0], role))
continue
LOG.debug("Network role candidates: {}, role: {},"
"inventory provider: {}".format(
region_ids, role,
self.ip_ext_manager.names()[0]))
role_candidates[role] = region_ids
# find candidates that meet conditions
for candidate in candidate_list:
# perform this check only for cloud candidates
if candidate["inventory_type"] != "cloud":
continue
c_any = False
c_all = True
for role in role_list:
if role not in role_candidates:
c_all = False
continue
rc = role_candidates.get(role)
if rc and candidate.get("candidate_id") not in rc:
c_all = False
# discard even if one role is not met
elif rc and candidate.get("candidate_id") in rc:
c_any = True
# include if any one role is met
if role_condition == 'any' and not c_any:
discard_set.add(candidate.get("candidate_id"))
elif role_condition == 'all' and not c_all:
discard_set.add(candidate.get("candidate_id"))
elif attrib == 'replication_role':
for candidate in candidate_list:
host_id = candidate.get("host_id")
if host_id:
results = self.ip_ext_manager.map_method(
'check_candidate_role', host_id=host_id)
if not results or len(results) < 1:
LOG.error(
_LE("Empty response for replication roles {}").
format(role))
discard_set.add(candidate.get("candidate_id"))
continue
# compare results from A&AI with the value in attribute constraint
if value and results[0] != value.upper():
discard_set.add(candidate.get("candidate_id"))
elif attrib == 'complex':
v_discard_set = \
self.get_candidate_discard_set(
value=value,
candidate_list=candidate_list,
value_attrib="complex_name")
discard_set.update(v_discard_set)
elif attrib == "country":
v_discard_set = \
self.get_candidate_discard_set(
value=value,
candidate_list=candidate_list,
value_attrib="country")
discard_set.update(v_discard_set)
elif attrib == "state":
v_discard_set = \
self.get_candidate_discard_set(
value=value,
candidate_list=candidate_list,
value_attrib="state")
discard_set.update(v_discard_set)
elif attrib == "region":
v_discard_set = \
self.get_candidate_discard_set(
value=value,
candidate_list=candidate_list,
value_attrib="region")
discard_set.update(v_discard_set)
elif attrib == "cloud-region":
v_discard_set = \
self.get_candidate_discard_set_by_cloud_region(
value=value,
candidate_list=candidate_list,
value_attrib="location_id")
discard_set.update(v_discard_set)
# return candidates not in discard set
candidate_list[:] = [
c for c in candidate_list if c['candidate_id'] not in discard_set
]
LOG.info("Available candidates after attribute checks: {}, "
"inventory provider: {}".format(
candidate_list,
self.ip_ext_manager.names()[0]))
return {'response': candidate_list, 'error': False}
def request(self,
method='get',
content_type='application/json',
path='',
headers=None,
data=None):
"""Performs HTTP request. Returns a requests.Response object."""
if method not in ('post', 'get', 'put', 'delete'):
method = 'get'
method_fn = getattr(self.session, method)
full_headers = {
'Accept': content_type,
'Content-Type': content_type,
}
if headers:
full_headers.update(headers)
full_url = '{}/{}'.format(self.server_url,
path.lstrip('/').rstrip('/'))
# Prepare the request args
try:
data_str = json.dumps(data) if data else None
except (TypeError, ValueError):
data_str = data
kwargs = {
'data': data_str,
'headers': full_headers,
'timeout': self.timeout,
'cert': (self.cert, self.key),
'verify': self.verify,
'stream': False,
}
if self.username or self.password:
LOG.debug("Using HTTPBasicAuth")
kwargs['auth'] = HTTPBasicAuth(self.username, self.password)
if self.cert and self.key:
LOG.debug("Using SSL/TLS Certificate/Key")
if self.log_debug:
LOG.debug("Request: {} {}".format(method.upper(), full_url))
if data:
LOG.debug("Request Body: {}".format(json.dumps(data)))
response = None
for attempt in range(self.retries):
if attempt > 0:
# No need to show 400 bad requests from Music - Ignorable when lock cannot be received at one particular point in time
if "MUSIC" not in full_url:
LOG.warn(
_LW("Retry #{}/{}").format(attempt + 1, self.retries))
try:
response = method_fn(full_url, **kwargs)
# We shouldn't have to do this since stream is set to False,
# but we're gonna anyway. See "Body Content Workflow" here:
# http://docs.python-requests.org/en/master/user/advanced/
response.close()
if not response.ok:
# No need to show 400 bad requests from Music - Ignorable when lock cannot be received at one particular point in time
if "MUSIC" not in full_url:
LOG.warn("Response Status: {} {}".format(
response.status_code, response.reason))
if self.log_debug and response.text:
try:
response_dict = json.loads(response.text)
LOG.debug("Response JSON: {}".format(
json.dumps(response_dict)))
except ValueError:
LOG.debug("Response Body: {}".format(response.text))
#response._content = response._content.decode()
if response.ok:
break
except requests.exceptions.RequestException as err:
LOG.error("Exception: %s", err.args)
# Response.__bool__ returns false if status is not ok. Ruh roh!
# That means we must check the object type vs treating it as a bool.
# More info: https://github.com/kennethreitz/requests/issues/2002
if isinstance(response, requests.models.Response) and not response.ok:
# No need to show 400 bad requests from Music - Ignorable when lock cannot be received at one particular point in time
if "MUSIC" not in full_url:
LOG.error(
_LE("Status {} {} after {} retries for URL: {}").format(
response.status_code, response.reason, self.retries,
full_url))
return response
def run(self):
"""Run"""
LOG.debug("{}".format(self.__class__.__name__))
# TODO(snarayanan): This is really meant to be a control loop
# As long as self.running is true, we process another request.
while self.running:
# Delay time (Seconds) for MUSIC requests.
time.sleep(self.conf.delay_time)
# plans = Plan.query().all()
# Find the first plan with a status of TRANSLATED.
# Change its status to SOLVING.
# Then, read the "translated" field as "template".
json_template = None
p = None
requests_to_solve = dict()
regions_maps = dict()
country_groups = list()
# Instead of using the query.all() method, now creating an index for 'status'
# field in conductor.plans table, and query plans by status columns
translated_plans = self.Plan.query.get_plan_by_col("status", self.Plan.TRANSLATED)
solving_plans = self.Plan.query.get_plan_by_col("status", self.Plan.SOLVING)
# combine the plans with status = 'translated' and 'solving' together
plans = translated_plans + solving_plans
found_translated_template = False
for p in plans:
if p.status == self.Plan.TRANSLATED:
json_template = p.translation
found_translated_template = True
break
elif p.status == self.Plan.SOLVING and (self.current_time_seconds()
- self.millisec_to_sec(p.updated)) > self.conf.solver.timeout:
p.status = self.Plan.TRANSLATED
p.update(condition=self.solving_status_condition)
break
if not json_template:
if found_translated_template:
message = _LE("Plan {} status is translated, yet "
"the translation wasn't found").format(p.id)
LOG.error(message)
p.status = self.Plan.ERROR
p.message = message
p.update(condition=self.translated_status_condition)
continue
if found_translated_template and p and p.solver_counter >= self.conf.solver.max_solver_counter:
message = _LE("Tried {} times. Plan {} is unable to solve").format(self.conf.solver.max_solver_counter,
p.id)
LOG.error(message)
p.status = self.Plan.ERROR
p.message = message
p.update(condition=self.translated_status_condition)
continue
log_util.setLoggerFilter(LOG, self.conf.keyspace, p.id)
p.status = self.Plan.SOLVING
p.solver_counter += 1
p.solver_owner = socket.gethostname()
_is_updated = p.update(condition=self.translated_status_condition)
if not _is_updated:
continue
# other VMs have updated the status and start solving the plan
if 'FAILURE' in _is_updated:
continue
LOG.info(_LI("Sovling starts, changing the template status from translated to solving, "
"atomic update response from MUSIC {}").format(_is_updated))
LOG.info(_LI("Plan {} with request id {} is solving by machine {}. Tried to solve it for {} times.").
format(p.id, p.name, p.solver_owner, p.solver_counter))
_is_success = "FAILURE"
request = parser.Parser()
request.cei = self.cei
request.request_id = p.name
request.plan_id = p.id
# getting the number of solutions need to provide
num_solution = getattr(p, 'recommend_max', '1')
if num_solution.isdigit():
num_solution = int(num_solution)
# TODO(inam/larry): move this part of logic inside of parser and don't apply it to distance_between
try:
# getting region placeholders from database and insert/put into regions_maps dictionary
region_placeholders = self.RegionPlaceholders.query.all()
for region in region_placeholders:
regions_maps.update(region.countries)
# getting country groups from database and insert into the country_groups list
customer_loc = ''
location_list = json_template["conductor_solver"]["locations"]
for location_id, location_info in location_list.items():
customer_loc = location_info['country']
countries = self.CountryLatency.query.get_plan_by_col("country_name", customer_loc)
LOG.info("Customer Location for Latency Reduction " + customer_loc)
if len(countries) == 0:
LOG.info("country is not present is country latency table, looking for * wildcard entry")
countries = self.CountryLatency.query.get_plan_by_col("country_name", "*")
if len(countries) != 0:
LOG.info("Found '*' wild card entry in country latency table")
else:
msg = "No '*' wild card entry found in country latency table. No solution will be provided"
LOG.info(msg)
p.message = msg
for country in countries:
country_groups = country.groups
LOG.info("Done getting Latency Country DB Groups ")
except Exception as error_msg:
LOG.error("Exception thrown while reading region_placeholders and country groups information "
"from database. Exception message: {}".format(error_msg))
try:
request.parse_template(json_template, country_groups, regions_maps)
request.assgin_constraints_to_demands()
requests_to_solve[p.id] = request
opt = optimizer.Optimizer(self.conf, _requests=requests_to_solve)
solution_list = opt.get_solution(num_solution)
except Exception as err:
message = _LE("Plan {} status encountered a "
"parsing error: {}").format(p.id, err)
LOG.error(traceback.print_exc())
p.status = self.Plan.ERROR
p.message = message
while 'FAILURE' in _is_success:
_is_success = p.update(condition=self.solver_owner_condition)
LOG.info(_LI("Encountered a parsing error, changing the template status from solving to error, "
"atomic update response from MUSIC {}").format(_is_success))
continue
LOG.info("Preparing the recommendations ")
# checking if the order is 'initial' or 'speed changed' one
is_speed_change = False
if request and request.request_type == 'speed changed':
is_speed_change = True
recommendations = []
if not solution_list or len(solution_list) < 1:
# when order takes too much time to solve
if (int(round(time.time())) - self.millisec_to_sec(p.updated)) > self.conf.solver.solver_timeout:
message = _LI("Plan {} is timed out, exceed the expected "
"time {} seconds").format(p.id, self.conf.solver.timeout)
# when no solution found
else:
message = _LI("Plan {} search failed, no "
"recommendations found by machine {}").format(p.id, p.solver_owner)
LOG.info(message)
# Update the plan status
p.status = self.Plan.NOT_FOUND
p.message = message
# Metrics to Prometheus
m_svc_name = p.template.get('parameters', {}).get('service_name', 'N/A')
PC.VNF_FAILURE.labels('ONAP', m_svc_name).inc()
while 'FAILURE' in _is_success:
_is_success = p.update(condition=self.solver_owner_condition)
LOG.info(_LI("Plan serach failed, changing the template status from solving to not found, "
"atomic update response from MUSIC {}").format(_is_success))
else:
# Assemble recommendation result JSON
for solution in solution_list:
current_rec = dict()
for demand_name in solution:
resource = solution[demand_name]
if not is_speed_change:
is_rehome = "false"
else:
is_rehome = "false" if resource.get("existing_placement") == 'true' else "true"
location_id = "" if resource.get("cloud_region_version") == '2.5' \
else resource.get("location_id")
rec = {
# FIXME(shankar) A&AI must not be hardcoded here.
# Also, account for more than one Inventory Provider.
"inventory_provider": "aai",
"service_resource_id":
resource.get("service_resource_id"),
"candidate": {
"candidate_id": resource.get("candidate_id"),
"inventory_type": resource.get("inventory_type"),
"cloud_owner": resource.get("cloud_owner"),
"location_type": resource.get("location_type"),
"location_id": location_id,
"is_rehome": is_rehome},
"attributes": {
"physical-location-id":
resource.get("physical_location_id"),
"cloud_owner": resource.get("cloud_owner"),
'aic_version': resource.get("cloud_region_version")},
}
if rec["candidate"]["inventory_type"] in ["nssi", "nsi", "slice_profiles"]:
rec["candidate"] = resource
if resource.get('vim-id'):
rec["candidate"]['vim-id'] = resource.get('vim-id')
if rec["candidate"]["inventory_type"] == "service":
rec["attributes"]["host_id"] = resource.get("host_id")
rec["attributes"]["service_instance_id"] = resource.get("candidate_id")
rec["candidate"]["host_id"] = resource.get("host_id")
if resource.get('vlan_key'):
rec["attributes"]['vlan_key'] = resource.get('vlan_key')
if resource.get('port_key'):
rec["attributes"]['port_key'] = resource.get('port_key')
if rec["candidate"]["inventory_type"] == "vfmodule":
rec["attributes"]["host_id"] = resource.get("host_id")
rec["attributes"]["service_instance_id"] = resource.get("service_instance_id")
rec["candidate"]["host_id"] = resource.get("host_id")
if resource.get('vlan_key'):
rec["attributes"]['vlan_key'] = resource.get('vlan_key')
if resource.get('port_key'):
rec["attributes"]['port_key'] = resource.get('port_key')
vf_module_data = rec["attributes"]
vf_module_data['nf-name'] = resource.get("nf-name")
vf_module_data['nf-id'] = resource.get("nf-id")
vf_module_data['nf-type'] = resource.get("nf-type")
vf_module_data['vnf-type'] = resource.get("vnf-type")
vf_module_data['vf-module-id'] = resource.get("vf-module-id")
vf_module_data['vf-module-name'] = resource.get("vf-module-name")
vf_module_data['ipv4-oam-address'] = resource.get("ipv4-oam-address")
vf_module_data['ipv6-oam-address'] = resource.get("ipv6-oam-address")
vf_module_data['vservers'] = resource.get("vservers")
elif rec["candidate"]["inventory_type"] == "cloud":
if resource.get("all_directives") and resource.get("flavor_map"):
rec["attributes"]["directives"] = \
self.set_flavor_in_flavor_directives(
resource.get("flavor_map"), resource.get("all_directives"))
# Metrics to Prometheus
m_vim_id = resource.get("vim-id")
m_hpa_score = resource.get("hpa_score", 0)
m_svc_name = p.template['parameters'].get(
'service_name', 'N/A')
for vnfc, flavor in resource.get("flavor_map").items():
PC.VNF_COMPUTE_PROFILES.labels('ONAP',
m_svc_name,
demand_name,
vnfc,
flavor,
m_vim_id).inc()
PC.VNF_SCORE.labels('ONAP', m_svc_name,
demand_name,
m_hpa_score).inc()
if resource.get('conflict_id'):
rec["candidate"]["conflict_id"] = resource.get("conflict_id")
if resource.get('passthrough_attributes'):
for key, value in resource.get('passthrough_attributes').items():
if key in rec["attributes"]:
LOG.error('Passthrough attribute {} in demand {} already exist for candidate {}'.
format(key, demand_name, rec['candidate_id']))
else:
rec["attributes"][key] = value
# TODO(snarayanan): Add total value to recommendations?
# msg = "--- total value of decision = {}"
# LOG.debug(msg.format(_best_path.total_value))
# msg = "--- total cost of decision = {}"
# LOG.debug(msg.format(_best_path.total_cost))
current_rec[demand_name] = rec
recommendations.append(current_rec)
# Update the plan with the solution
p.solution = {
"recommendations": recommendations
}
# multiple spin-ups logic
'''
go through list of recommendations in the solution
for cloud candidates, check if (cloud-region-id + e2evnfkey) is in the order_locks table
if so, insert the row with status 'parked' in order_locks, changes plan status to 'pending' in plans
table (or other status value)
otherwise, insert the row with status 'locked' in order_locks, and change status to 'solved' in plans
table - continue reservation
'''
# clean up the data/record in order_locks table, deleting all records that failed from MSO
order_locks = self.OrderLock.query.all()
for order_lock_record in order_locks:
plans = getattr(order_lock_record, 'plans')
for plan_id, plan_attributes in plans.items():
plan_dict = json.loads(plan_attributes)
if plan_dict.get('status', None) == OrderLock.FAILED:
order_lock_record.delete()
LOG.info(_LI("The order lock record {} with status {} is deleted (due to failure"
" spinup from MSO) from order_locks table").
format(order_lock_record, plan_dict.get('status')))
break
inserted_order_records_dict = dict()
available_dependenies_set = set()
is_inserted_to_order_locks = True
is_conflict_id_missing = False
is_order_translated_before_spinup = False
for solution in solution_list:
for demand_name, candidate in solution.items():
if candidate.get('inventory_type') == 'cloud':
conflict_id = candidate.get('conflict_id')
service_resource_id = candidate.get('service_resource_id')
# TODO(larry): add more logic for missing conflict_id in template
if not conflict_id:
is_conflict_id_missing = True
break
available_dependenies_set.add(conflict_id)
# check if conflict_id exists in order_locks table
order_lock_record = self.OrderLock.query.get_plan_by_col("id", conflict_id)
if order_lock_record:
is_spinup_completed = getattr(order_lock_record[0], 'is_spinup_completed')
spinup_completed_timestamp = getattr(order_lock_record[0],
'spinup_completed_timestamp')
if is_spinup_completed and spinup_completed_timestamp > p.translation_begin_timestamp:
is_order_translated_before_spinup = True
break
elif not is_spinup_completed:
inserted_order_records_dict[conflict_id] = service_resource_id
if is_conflict_id_missing:
message = _LE("Missing conflict identifier field for cloud candidates in the template, "
"could not insert into order_locks table")
LOG.debug(message)
p.status = self.Plan.SOLVED
elif is_order_translated_before_spinup:
message = _LE("Retriggering Plan {} due to the new order arrives before the "
"spinup completion of the old order ").format(p.id)
LOG.debug(message)
p.rehome_plan()
elif len(inserted_order_records_dict) > 0:
new_dependenies_set = available_dependenies_set - set(inserted_order_records_dict.keys())
dependencies = ','.join(str(s) for s in new_dependenies_set)
for conflict_id, service_resource_id in inserted_order_records_dict.items():
plan = {
p.id: {
"status": OrderLock.UNDER_SPIN_UP,
"created": self.current_time_millis(),
"updated": self.current_time_millis(),
"service_resource_id": service_resource_id
}
}
if dependencies:
plan[p.id]['dependencies'] = dependencies
order_lock_row = self.OrderLock(id=conflict_id, plans=plan)
response = order_lock_row.insert()
# TODO(larry): add more logs for inserting order lock record (insert/update)
LOG.info(_LI("Inserting the order lock record to order_locks table in MUSIC, "
"conditional insert operation response from MUSIC {}").format(response))
if response and response.status_code == 200:
body = response.json()
LOG.info("Succcessfully inserted the record in order_locks table with "
"the following response message {}".format(body))
else:
is_inserted_to_order_locks = False
else:
for solution in solution_list:
for demand_name, candidate in solution.items():
if candidate.get('inventory_type') == 'cloud':
conflict_id = candidate.get('conflict_id')
service_resource_id = candidate.get('service_resource_id')
order_lock_record = self.OrderLock.query.get_plan_by_col("id", conflict_id)
if order_lock_record:
deleting_record = order_lock_record[0]
plans = getattr(deleting_record, 'plans')
is_spinup_completed = getattr(deleting_record, 'is_spinup_completed')
spinup_completed_timestamp = getattr(deleting_record, 'spinup_completed_timestamp')
if is_spinup_completed:
# persist the record in order_locks_history table
order_lock_history_record = \
self.OrderLockHistory(conflict_id=conflict_id, plans=plans,
is_spinup_completed=is_spinup_completed,
spinup_completed_timestamp=spinup_completed_timestamp
)
LOG.debug("Inserting the history record with conflict id {}"
" to order_locks_history table".format(conflict_id))
order_lock_history_record.insert()
# remove the older record
LOG.debug("Deleting the order lock record {} from order_locks table"
.format(deleting_record))
deleting_record.delete()
plan = {
p.id: {
"status": OrderLock.UNDER_SPIN_UP,
"created": self.current_time_millis(),
"updated": self.current_time_millis(),
"service_resource_id": service_resource_id
}
}
order_lock_row = self.OrderLock(id=conflict_id, plans=plan)
response = order_lock_row.insert()
# TODO(larry): add more logs for inserting order lock record (insert/update)
LOG.info(_LI("Inserting the order lock record to order_locks table in MUSIC, "
"conditional insert operation response from MUSIC {}").format(response))
if response and response.status_code == 200:
body = response.json()
LOG.info("Succcessfully inserted the record in order_locks table "
"with the following response message {}".format(body))
else:
is_inserted_to_order_locks = False
if not is_inserted_to_order_locks:
message = _LE("Plan {} status encountered an "
"error while inserting order lock message to MUSIC.").format(p.id)
LOG.error(message)
p.status = self.Plan.ERROR
p.message = message
elif p.status == self.Plan.SOLVING:
if len(inserted_order_records_dict) > 0:
LOG.info(_LI("The plan with id {} is parked in order_locks table,"
"waiting for MSO release calls").format(p.id))
p.status = self.Plan.WAITING_SPINUP
else:
LOG.info(_LI("The plan with id {} is inserted in order_locks table.").
format(p.id))
p.status = self.Plan.SOLVED
while 'FAILURE' in _is_success \
and (self.current_time_seconds() - self.millisec_to_sec(p.updated)) <= self.conf.solver.timeout:
_is_success = p.update(condition=self.solver_owner_condition)
LOG.info(_LI("Plan search complete, changing the template status from solving to {}, "
"atomic update response from MUSIC {}").format(p.status, _is_success))
LOG.info(_LI("Plan {} search complete, {} solution(s) found by machine {}").
format(p.id, len(solution_list), p.solver_owner))
LOG.debug("Plan {} detailed solution: {}".
format(p.id, p.solution))
LOG.info("Plan name: {}".format(p.name))
def __set__(self, obj, value):
"""Set attribute"""
if not self.fset:
raise AttributeError(_LE("Can't set attribute"))
type_ = type(obj)
return self.fset.__get__(obj, type_)(value)
def run(self):
"""Run"""
LOG.debug("%s" % self.__class__.__name__)
# TODO(snarayanan): This is really meant to be a control loop
# As long as self.running is true, we process another request.
while self.running:
# Delay time (Seconds) for MUSIC requests.
time.sleep(self.conf.delay_time)
# plans = Plan.query().all()
# Find the first plan with a status of SOLVED.
# Change its status to RESERVING.
solution = None
translation = None
p = None
# requests_to_reserve = dict()
# Instead of using the query.all() method, now creating an index for 'status'
# field in conductor.plans table, and query plans by status columns
solved_plans = self.Plan.query.get_plan_by_col(
"status", self.Plan.SOLVED)
reserving_plans = self.Plan.query.get_plan_by_col(
"status", self.Plan.RESERVING)
# combine the plans with status = 'solved' and 'reserving' together
plans = solved_plans + reserving_plans
found_solved_template = False
for p in plans:
# when a plan is in RESERVING status more than timeout value
if p.status == self.Plan.RESERVING and \
(self.current_time_seconds() - self.millisec_to_sec(p.updated)) > self.conf.reservation.timeout:
# change the plan status to SOLVED for another VM to reserve
p.status = self.Plan.SOLVED
p.update(condition=self.reservating_status_condition)
break
elif p.status == self.Plan.SOLVED:
solution = p.solution
translation = p.translation
found_solved_template = True
break
if not solution:
if found_solved_template:
message = _LE("Plan {} status is solved, yet "
"the solution wasn't found").format(p.id)
LOG.error(message)
p.status = self.Plan.ERROR
p.message = message
p.update(condition=self.solved_status_condition)
continue
if found_solved_template and p and p.reservation_counter >= self.conf.reservation.max_reservation_counter:
message = _LE("Tried {} times. Plan {} is unable to reserve") \
.format(self.conf.reservation.max_reservation_counter, p.id)
LOG.error(message)
p.status = self.Plan.ERROR
p.message = message
p.update(condition=self.solved_status_condition)
continue
log_util.setLoggerFilter(LOG, self.conf.keyspace, p.id)
# update status to reserving
p.status = self.Plan.RESERVING
p.reservation_counter += 1
p.reservation_owner = socket.gethostname()
_is_updated = p.update(condition=self.solved_status_condition)
if not _is_updated:
continue
if 'FAILURE' in _is_updated:
continue
LOG.info(
_LI("Reservation starts, changing the template status from solved to reserving, "
"atomic update response from MUSIC {}").format(
_is_updated))
LOG.info(
_LI("Plan {} with request id {} is reserving by machine {}. Tried to reserve it for {} times."
).format(p.id, p.name, p.reservation_owner,
p.reservation_counter))
# begin reservations
# if plan needs reservation proceed with reservation
# else set status to done.
reservations = None
_is_success = "FAILURE"
if translation:
conductor_solver = translation.get("conductor_solver")
if conductor_solver:
reservations = conductor_solver.get("reservations")
else:
LOG.error("no conductor_solver in "
"translation for Plan {}".format(p.id))
if reservations:
recommendations = solution.get("recommendations")
reservation_list = list()
# TODO(larry) combine the two reservation logic as one, make the code service independent
sdwan_candidate_list = list()
service_model = reservations.get("service_model")
for reservation, resource in reservations.get("demands",
{}).items():
candidates = list()
reservation_demand = resource.get("demands")
reservation_name = resource.get("name")
reservation_type = resource.get("type")
reservation_properties = resource.get("properties")
if reservation_properties:
controller = reservation_properties.get("controller")
request = reservation_properties.get("request")
for recommendation in recommendations:
for demand, r_resource in recommendation.items():
if demand == reservation_demand:
# get selected candidate from translation
selected_candidate_id = r_resource.get(
"candidate").get("candidate_id")
demands = translation.get(
"conductor_solver").get("demands")
for demand_name, d_resource in demands.items():
if demand_name == demand:
for candidate in d_resource.get(
"candidates"):
if candidate.get(
"candidate_id"
) == selected_candidate_id:
candidate['request'] = request
candidates.append(candidate)
sdwan_candidate_list.append(
candidate)
#TODO(larry) combine the two reservation logic as one, make the code service independent
if service_model == "ADIOD":
is_success = self.try_reservation_call(
method="reserve",
candidate_list=candidates,
reservation_type=service_model,
controller=controller,
request=request,
reservation_name=None)
# if reservation succeed continue with next candidate
if is_success:
curr_reservation = dict()
curr_reservation['candidate_list'] = candidates
curr_reservation[
'reservation_name'] = reservation_name
curr_reservation[
'reservation_type'] = reservation_type
curr_reservation['controller'] = controller
curr_reservation['request'] = request
reservation_list.append(curr_reservation)
else:
# begin roll back of all reserved resources on
# the first failed reservation
rollback_status = \
self.rollback_reservation(reservation_list)
# order_lock spin-up rollback
for decision in solution.get('recommendations'):
candidate = list(decision.values())[0].get(
'candidate'
) # Python 3 Conversion -- dict object to list object
if candidate.get('inventory_type') == 'cloud':
# TODO(larry) change the code to get('conflict_id') instead of 'location_id'
conflict_id = candidate.get('conflict_id')
order_record = self.OrderLock.query.get_plan_by_col(
"id", conflict_id)[0]
if order_record:
order_record.delete()
# statuses
if rollback_status:
# released all reservations,
# move plan to translated
if p.reservation_counter >= self.conf.reservation.max_reservation_counter:
p.status = self.Plan.ERROR
p.message = _LE(
"Tried {} times. Plan {} is unable to reserve"
).format(
self.conf.reservation.
max_reservation_counter, p.id)
LOG.error(p.message)
else:
p.status = self.Plan.TRANSLATED
# TODO(larry): Should be replaced by the new api from MUSIC
while 'FAILURE' in _is_success:
_is_success = p.update(
condition=self.
reservation_owner_condition)
LOG.info(
_LI("Rolling back the template from reserving to {} status, "
"atomic update response from MUSIC {}"
).format(p.status, _is_success))
del reservation_list[:]
else:
LOG.error("Reservation rollback failed")
p.status = self.Plan.ERROR
p.message = "Reservation release failed"
# TODO(larry): Should be replaced by the new api from MUSIC
while 'FAILURE' in _is_success:
_is_success = p.update(
condition=self.
reservation_owner_condition)
LOG.info(
_LI("Rollback Failed, Changing the template status from reserving to error, "
"atomic update response from MUSIC {}"
).format(_is_success))
break # reservation failed
continue
# continue with reserving the next candidate
# TODO(larry) combine the two reservation logic as one, make the code service independent
if service_model == "DHV":
is_success = self.try_reservation_call(
method="reserve",
candidate_list=sdwan_candidate_list,
reservation_type=service_model,
controller=controller,
request=request,
reservation_name=None)
if not is_success:
# order_lock spin-up rollback
for decision in solution.get('recommendations'):
candidate = list(decision.values())[0].get(
'candidate'
) # Python 3 Conversion -- dict object to list object
if candidate.get('inventory_type') == 'cloud':
conflict_id = candidate.get('conflict_id')
order_record = self.OrderLock.query.get_plan_by_col(
"id", conflict_id)[0]
if order_record:
order_record.delete()
if p.reservation_counter >= self.conf.reservation.max_reservation_counter:
p.status = self.Plan.ERROR
p.message = _LE(
"Tried {} times. Plan {} is unable to reserve"
).format(
self.conf.reservation.max_reservation_counter,
p.id)
LOG.error(p.message)
else:
p.status = self.Plan.TRANSLATED
# TODO(larry): Should be replaced by the new api from MUSIC
while 'FAILURE' in _is_success:
_is_success = p.update(
condition=self.reservation_owner_condition)
LOG.info(
_LI("Rolling back the template from reserving to {} status, "
"atomic update response from MUSIC {}").
format(p.status, _is_success))
del reservation_list[:]
# verify if all candidates have been reserved
if p.status == self.Plan.RESERVING:
# all reservations succeeded.
LOG.info(
_LI("Plan {} with request id {} Reservation complete").
format(p.id, p.name))
LOG.debug("Plan {} Reservation complete".format(p.id))
p.status = self.Plan.DONE
while 'FAILURE' in _is_success and (
self.current_time_seconds() - self.millisec_to_sec(
p.updated)) <= self.conf.reservation.timeout:
_is_success = p.update(
condition=self.reservation_owner_condition)
LOG.info(
_LI("Reservation is complete, changing the template status from reserving to done, "
"atomic update response from MUSIC {}").format(
_is_success))
continue
def __check_for_templates(self):
"""Wait for the polling interval, then do the real template check."""
# Wait for at least poll_interval sec
polling_interval = self.conf.controller.polling_interval
time.sleep(polling_interval)
# Look for plans with the status set to TEMPLATE
# Instead of using the query.all() method, now creating an index for 'status'
# field in conductor.plans table, and query plans by status columns
template_plans = self.Plan.query.get_plan_by_col(
"status", self.Plan.TEMPLATE)
translating_plans = self.Plan.query.get_plan_by_col(
"status", self.Plan.TRANSLATING)
# combine the plans with status = 'template' and 'translating' together
plans = template_plans + translating_plans
for plan in plans:
# If there's a template to be translated, do it!
if plan.status == self.Plan.TEMPLATE:
if plan.translation_counter >= self.conf.controller.max_translation_counter:
message = _LE("Tried {} times. Plan {} is unable to translate") \
.format(self.conf.controller.max_translation_counter, plan.id)
plan.message = message
plan.status = self.Plan.ERROR
plan.update(condition=self.template_status_condition)
LOG.error(message)
break
else:
# change the plan status to "translating" and assign the current machine as translation owner
plan.status = self.Plan.TRANSLATING
plan.translation_counter += 1
plan.translation_owner = socket.gethostname()
plan.translation_begin_timestamp = int(
round(time.time() * 1000))
_is_updated = plan.update(
condition=self.template_status_condition)
log_util.setLoggerFilter(LOG, self.conf.keyspace, plan.id)
LOG.info(
_LE("Plan {} is trying to update the status from 'template' to 'translating',"
" get {} response from MUSIC").format(
plan.id, _is_updated))
if not _is_updated:
break
if _is_updated and 'SUCCESS' in _is_updated:
self.translate(plan)
break
# TODO(larry): sychronized clock among Conducotr VMs, or use an offset
elif plan.status == self.Plan.TRANSLATING and (self.current_time_seconds()
- self.millisec_to_sec(plan.updated)) \
> self.conf.messaging_server.timeout:
plan.status = self.Plan.TEMPLATE
plan.update(condition=self.translating_status_condition)
break
elif plan.timedout:
# TODO(jdandrea): How to tell all involved to stop working?
# Not enough to just set status.
continue
def _do(self):
"""Look for a new RPC call and serve it"""
# Get all the messages in queue
msgs = self.RPC.query.all()
for msg in msgs:
# Find the first msg marked as enqueued.
if msg.working and \
(self.current_time_seconds() - self.millisec_to_sec(msg.updated))\
> self.conf.messaging_server.response_timeout:
msg.status = message.Message.ENQUEUED
msg.update(condition=self.working_status_condition)
if not msg.enqueued:
continue
if 'plan_name' in msg.ctxt.keys():
LOG.info('Plan name: {}'.format(msg.ctxt['plan_name']))
elif 'plan_name' in msg.args.keys():
LOG.info('Plan name: {}'.format(msg.args['plan_name']))
# Change the status to WORKING (operation with a lock)
msg.status = message.Message.WORKING
msg.owner = socket.gethostname()
# All update should have a condition (status == enqueued)
_is_updated = msg.update(condition=self.enqueued_status_condition)
if not _is_updated or 'FAILURE' in _is_updated:
continue
# RPC methods must not start/end with an underscore.
if msg.method.startswith('_') or msg.method.endswith('_'):
error_msg = _LE("Method {} must not start or end"
"with underscores").format(msg.method)
self._log_error_and_update_msg(msg, error_msg)
return
# The first endpoint that supports the method wins.
method = None
for endpoint in self.endpoints:
if msg.method not in dir(endpoint):
continue
endpoint_method = getattr(endpoint, msg.method)
if callable(endpoint_method):
method = endpoint_method
if self.conf.messaging_server.debug:
LOG.debug("Message {} method {} is "
"handled by endpoint {}".format(
msg.id, msg.method,
method.__str__.__name__))
break
if not method:
error_msg = _LE("Message {} method {} unsupported "
"in endpoints.").format(msg.id, msg.method)
self._log_error_and_update_msg(msg, error_msg)
return
# All methods must take a ctxt and args param.
if inspect.getargspec(method).args != ['self', 'ctx', 'arg']:
error_msg = _LE("Method {} must take three args: "
"self, ctx, arg").format(msg.method)
self._log_error_and_update_msg(msg, error_msg)
return
LOG.info(
_LI("Message {} method {} received").format(
msg.id, msg.method))
if self.conf.messaging_server.debug:
LOG.debug(
_LI("Message {} method {} context: {}, args: {}").format(
msg.id, msg.method, msg.ctxt, msg.args))
failure = None
try:
# Add the template to conductor.plan table
# Methods return an opaque dictionary
result = method(msg.ctxt, msg.args)
# FIXME(jdandrea): Remove response/error and make it opaque.
# That means this would just be assigned result outright.
msg.response = result.get('response', result)
except Exception:
# Current sys.exc_info() content can be overridden
# by another exception raised by a log handler during
# LOG.exception(). So keep a copy and delete it later.
failure = sys.exc_info()
# Do not log details about the failure here. It will
# be returned later upstream.
LOG.exception(_LE('Exception during message handling'))
try:
if failure is None:
msg.status = message.Message.COMPLETED
else:
msg.failure = \
rpc_common.serialize_remote_exception(failure)
msg.status = message.Message.ERROR
LOG.info(
_LI("Message {} method {}, status: {}").format(
msg.id, msg.method, msg.status))
if self.conf.messaging_server.debug:
LOG.debug("Message {} method {}, response: {}".format(
msg.id, msg.method, msg.response))
_is_success = 'FAILURE'
while 'FAILURE' in _is_success and (
self.current_time_seconds() -
self.millisec_to_sec(msg.updated)
) <= self.conf.messaging_server.response_timeout:
_is_success = msg.update()
LOG.info(
_LI("updating the message status from working to {}, "
"atomic update response from MUSIC {}").format(
msg.status, _is_success))
except Exception:
LOG.exception(
_LE("Can not send reply for message {} "
"method {}").format(msg.id, msg.method))
finally:
# Remove circular object reference between the current
# stack frame and the traceback in exc_info.
del failure
def call(self, ctxt, method, args):
"""Synchronous Call"""
# # check if the call has a message saved in cache
# # key: string concatenation of ctxt + method + args
# # value: rpc response object
# key = ""
# for k, v in ctxt.items():
# key += str(k)
# key += '#' + str(v) + '#'
# key += '|' + str(method) + '|'
# for k, v in args.items():
# key += str(k)
# key += '#' + str(v) + '#'
#
# # check if the method has been called before
# # and cached
# if key in self.message_cache:
# LOG.debug("Retrieved method {} with args "
# "{} from cache".format(method, args))
# return self.message_cache[key]
rpc_start_time = time.time()
rpc = self.RPC(action=self.RPC.CALL,
ctxt=ctxt,
method=method,
args=args)
# TODO(jdandrea): Do something if the assert fails.
assert (rpc.enqueued)
rpc_id = rpc.id
topic = self.target.topic
LOG.info(_LI("Message {} on topic {} enqueued.").format(rpc_id, topic))
if self.conf.messaging_server.debug:
LOG.debug("Calling method {} with args {}".format(method, args))
# Check message status within a thread
executor = futurist.ThreadPoolExecutor()
started_at = time.time()
while (time.time() -
started_at) <= self.conf.messaging_server.response_timeout:
fut = executor.submit(self.__check_rpc_status, rpc_id, method)
rpc = fut.result()
if rpc and rpc.finished:
if self.conf.messaging_server.debug:
LOG.debug("Message {} method {} response received".format(
rpc_id, method))
break
executor.shutdown()
# Get response, delete message, and return response
if not rpc or not rpc.finished:
LOG.error(
_LE("Message {} on topic {} timed out at {} seconds").format(
rpc_id, topic,
self.conf.messaging_server.response_timeout))
elif not rpc.ok:
LOG.error(
_LE("Message {} on topic {} returned an error").format(
rpc_id, topic))
response = rpc.response
failure = rpc.failure
rpc.delete() # TODO(jdandrea): Put a TTL on the msg instead?
# self.message_cache[key] = response
LOG.debug("Elapsed time: {0:.3f} sec".format(time.time() -
rpc_start_time))
# If there's a failure, raise it as an exception
allowed = []
if failure is not None and failure != '':
# TODO(jdandrea): Do we need to populate allowed(_remote_exmods)?
raise rpc_common.deserialize_remote_exception(failure, allowed)
return response
