def get_scope_fields(field, references, req, policies):
"""Retrieve the values for scope fields from a request and policies
They are derived as per the configuration and references defined in a
configuration file. If the value of a scope field missing in a request or
policies, throw an exception since correct policies cannot be retrieved.
:param field: details on a scope field from a configuration file.
:param references: references defined in a configuration file.
:param req: an optimization request.
:param policy_info: a list of policies.
:return: scope fields retrieved from a request and policies.
"""
ref_source = references.get(field.get('get_param', ""), {}).get('source')
ref_value = references.get(field.get('get_param', ""), {}).get('value')
if ref_source == "request":
scope_field = dot_notation(req, ref_value)
if scope_field:
return scope_field
raise BusinessException(
"Field {} is missing a value in a request".format(
ref_value.split('.')[-1]))
else:
scope_fields = []
for policyName in policies.keys():
policy_content = policies.get(policyName)
if policy_content.get('type', "invalid_policy") == ref_source:
scope_fields.append(dot_notation(policy_content, ref_value))
scope_values = list_flatten(scope_fields)
if len(scope_values) > 0:
return scope_values
raise BusinessException(
"Field {} is missing a value in all policies of type {}".format(
ref_value.split('.')[-1], ref_source))
def is_valid_optim_request(request_json):
# Method to check whether the requestinfo/optimizer value is valid.
opt_info = request_json['optimInfo']
if not opt_info.get('modelId'):
if not opt_info.get('modelContent') or not opt_info.get('solver'):
raise BusinessException(
'modelContent and solver needs to be populated if model_id is not set'
)
if not opt_info.get('optData'):
raise BusinessException(
'optimInfo.optData needs to be populated to solve for a problem')
return True
def run_optimizer(request_json):
validate_request(request_json)
model_content, solver = get_model_content(request_json)
if solver == 'mzn':
return mzn_solve(request_json, model_content)
elif solver == 'py':
return py_solve(request_json, model_content)
raise BusinessException(
'Unsupported optimization solver requested {} '.format(solver))
def get_by_name(rest_client, policy_name_list, wildcards=True):
policy_list = []
for policy_name in policy_name_list:
try:
query_name = policy_name
if wildcards:
query_name = policy_name_as_regex(query_name)
policy_list.append(
rest_client.request(json={"policyName": query_name}))
except RequestException as err:
audit_log.warn("Error in fetching policy: " + policy_name)
raise BusinessException(
"Cannot fetch policy {}: ".format(policy_name), err)
return policy_list
def get_model_content(request_json):
model_id = request_json['optimInfo'].get('modelId')
if model_id:
status, data = get_model_data(model_id)
if status == 200:
model_content = decode_data(data[1])
solver = data[3]
else:
raise BusinessException(
'model_id [{}] not found in the model database'.format(
model_id))
else:
model_content = request_json['optimInfo']['modelContent']
solver = request_json['optimInfo']['solver']
return model_content, solver
def remote_api(req_json, osdf_config, service_type="placement"):
"""Make a request to policy and return response -- it accounts for multiple requests that be needed
:param req_json: policy request object (can have multiple policy names)
:param osdf_config: main config that will have credential information
:param service_type: the type of service to call: "placement", "scheduling"
:return: all related policies and provStatus retrieved from Subscriber policy
"""
config = osdf_config.deployment
headers = {"Content-type": "application/json"}
uid, passwd = config['policyPlatformUsername'], config[
'policyPlatformPassword']
url = config['policyPlatformUrl']
rc = RestClient(userid=uid,
passwd=passwd,
headers=headers,
url=url,
log_func=debug_log.debug)
if osdf_config.core['policy_info'][service_type][
'policy_fetch'] == "by_name":
policies = get_by_name(rc,
req_json[service_type + "Info"]['policyId'],
wildcards=True)
elif osdf_config.core['policy_info'][service_type][
'policy_fetch'] == "by_name_no_wildcards":
policies = get_by_name(rc,
req_json[service_type + "Info"]['policyId'],
wildcards=False)
else:
policies = get_by_scope(rc, req_json, osdf_config.core, service_type)
formatted_policies = []
for x in policies:
if x[list(x.keys())[0]].get('properties') is None:
raise BusinessException(
"Properties not found for policy with name %s" %
x[list(x.keys()[0])])
else:
formatted_policies.append(x)
return formatted_policies
def create_model_data(model_api):
with app.app_context():
try:
model_info = model_api['modelInfo']
model_id = model_info['modelId']
debug_log.debug("persisting model_api {}".format(model_id))
connection = get_db()
cursor = connection.cursor(buffered=True)
query = "SELECT model_id FROM optim_model_data WHERE model_id = %s"
values = (model_id, )
cursor.execute(query, values)
if cursor.fetchone() is None:
query = "INSERT INTO optim_model_data (model_id, model_content, description, solver_type) VALUES " \
"(%s, %s, %s, %s)"
values = (model_id, model_info['modelContent'],
model_info.get('description'), model_info['solver'])
cursor.execute(query, values)
g.pg.commit()
debug_log.debug(
"A record successfully inserted for request_id: {}".format(
model_id))
return retrieve_model_data(model_id)
close_db()
else:
query = "UPDATE optim_model_data SET model_content = %s, description = %s, solver_type = %s where " \
"model_id = %s "
values = (model_info['modelContent'],
model_info.get('description'), model_info['solver'],
model_id)
cursor.execute(query, values)
g.pg.commit()
return retrieve_model_data(model_id)
close_db()
except Exception as err:
error_log.error("error for request_id: {} - {}".format(
model_id, traceback.format_exc()))
close_db()
raise BusinessException(err)
def delete_model_data(model_id):
with app.app_context():
try:
debug_log.debug(
"deleting model data given model_id = {}".format(model_id))
connection = get_db()
cursor = connection.cursor(buffered=True)
query = "delete from optim_model_data WHERE model_id = %s"
values = (model_id, )
cursor.execute(query, values)
g.pg.commit()
close_db()
resp = {
"statusMessage":
"model data for modelId {} deleted".format(model_id)
}
return build_response(json.dumps(resp), 200)
except Exception as err:
error_log.error("error deleting model_id: {} - {}".format(
model_id, traceback.format_exc()))
close_db()
raise BusinessException(err)
def request(req_info, demands, request_parameters, service_info, template_fields,
osdf_config, flat_policies):
config = osdf_config.deployment
local_config = osdf_config.core
uid, passwd = config['conductorUsername'], config['conductorPassword']
conductor_url = config['conductorUrl']
req_id = req_info["requestId"]
transaction_id = req_info['transactionId']
headers = dict(transaction_id=transaction_id)
placement_ver_enabled = config.get('placementVersioningEnabled', False)
if placement_ver_enabled:
cond_minor_version = config.get('conductorMinorVersion', None)
if cond_minor_version is not None:
x_minor_version = str(cond_minor_version)
headers.update({'X-MinorVersion': x_minor_version})
debug_log.debug("Versions set in HTTP header to "
"conductor: X-MinorVersion: {} ".format(x_minor_version))
max_retries = config.get('conductorMaxRetries', 30)
ping_wait_time = config.get('conductorPingWaitTime', 60)
rc = RestClient(userid=uid, passwd=passwd, method="GET", log_func=debug_log.debug,
headers=headers)
conductor_req_json_str = conductor_api_builder(req_info, demands, request_parameters,
service_info, template_fields, flat_policies,
local_config)
conductor_req_json = json.loads(conductor_req_json_str)
debug_log.debug("Sending first Conductor request for request_id {}".format(req_id))
resp, raw_resp = initial_request_to_conductor(rc, conductor_url, conductor_req_json)
# Very crude way of keeping track of time.
# We are not counting initial request time, first call back, or time for HTTP request
total_time, ctr = 0, 2
client_timeout = req_info['timeout']
configured_timeout = max_retries * ping_wait_time
max_timeout = min(client_timeout, configured_timeout)
while True: # keep requesting conductor till we get a result or we run out of time
if resp is not None:
if resp["plans"][0].get("status") in ["error"]:
raise RequestException(response=raw_resp, request=raw_resp.request)
if resp["plans"][0].get("status") in ["done", "not found", "solved"]:
return resp
new_url = resp['plans'][0]['links'][0][0]['href'] # TODO(krishna): check why a list of lists
if total_time >= max_timeout:
raise BusinessException("Conductor could not provide a solution within {} seconds,"
"this transaction is timing out".format(max_timeout))
time.sleep(ping_wait_time)
ctr += 1
debug_log.debug("Attempt number {} url {}; prior status={}"
.format(ctr, new_url, resp['plans'][0]['status']))
total_time += ping_wait_time
try:
raw_resp = rc.request(new_url, raw_response=True)
resp = raw_resp.json()
except RequestException as e:
debug_log.debug("Conductor attempt {} for request_id {} has failed because {}"
.format(ctr, req_id, str(e)))
def test_handle_business_exception(self):
e = BusinessException("Business Exception Description")
resp = baseapp.handle_business_exception(e)
assert resp.status_code == 400
def validate_request(request_json):
OptimizationAPI(request_json).validate()
if not is_valid_optim_request(request_json):
raise BusinessException('Invalid optim request ')
return True