def get_scope_fields(field, references, req, policies): """Retrieve the values for scope fields from a request and policies They are derived as per the configuration and references defined in a configuration file. If the value of a scope field missing in a request or policies, throw an exception since correct policies cannot be retrieved. :param field: details on a scope field from a configuration file. :param references: references defined in a configuration file. :param req: an optimization request. :param policy_info: a list of policies. :return: scope fields retrieved from a request and policies. """ ref_source = references.get(field.get('get_param', ""), {}).get('source') ref_value = references.get(field.get('get_param', ""), {}).get('value') if ref_source == "request": scope_field = dot_notation(req, ref_value) if scope_field: return scope_field raise BusinessException( "Field {} is missing a value in a request".format( ref_value.split('.')[-1])) else: scope_fields = [] for policyName in policies.keys(): policy_content = policies.get(policyName) if policy_content.get('type', "invalid_policy") == ref_source: scope_fields.append(dot_notation(policy_content, ref_value)) scope_values = list_flatten(scope_fields) if len(scope_values) > 0: return scope_values raise BusinessException( "Field {} is missing a value in all policies of type {}".format( ref_value.split('.')[-1], ref_source))
def is_valid_optim_request(request_json): # Method to check whether the requestinfo/optimizer value is valid. opt_info = request_json['optimInfo'] if not opt_info.get('modelId'): if not opt_info.get('modelContent') or not opt_info.get('solver'): raise BusinessException( 'modelContent and solver needs to be populated if model_id is not set' ) if not opt_info.get('optData'): raise BusinessException( 'optimInfo.optData needs to be populated to solve for a problem') return True
def run_optimizer(request_json): validate_request(request_json) model_content, solver = get_model_content(request_json) if solver == 'mzn': return mzn_solve(request_json, model_content) elif solver == 'py': return py_solve(request_json, model_content) raise BusinessException( 'Unsupported optimization solver requested {} '.format(solver))
def get_by_name(rest_client, policy_name_list, wildcards=True): policy_list = [] for policy_name in policy_name_list: try: query_name = policy_name if wildcards: query_name = policy_name_as_regex(query_name) policy_list.append( rest_client.request(json={"policyName": query_name})) except RequestException as err: audit_log.warn("Error in fetching policy: " + policy_name) raise BusinessException( "Cannot fetch policy {}: ".format(policy_name), err) return policy_list
def get_model_content(request_json): model_id = request_json['optimInfo'].get('modelId') if model_id: status, data = get_model_data(model_id) if status == 200: model_content = decode_data(data[1]) solver = data[3] else: raise BusinessException( 'model_id [{}] not found in the model database'.format( model_id)) else: model_content = request_json['optimInfo']['modelContent'] solver = request_json['optimInfo']['solver'] return model_content, solver
def remote_api(req_json, osdf_config, service_type="placement"): """Make a request to policy and return response -- it accounts for multiple requests that be needed :param req_json: policy request object (can have multiple policy names) :param osdf_config: main config that will have credential information :param service_type: the type of service to call: "placement", "scheduling" :return: all related policies and provStatus retrieved from Subscriber policy """ config = osdf_config.deployment headers = {"Content-type": "application/json"} uid, passwd = config['policyPlatformUsername'], config[ 'policyPlatformPassword'] url = config['policyPlatformUrl'] rc = RestClient(userid=uid, passwd=passwd, headers=headers, url=url, log_func=debug_log.debug) if osdf_config.core['policy_info'][service_type][ 'policy_fetch'] == "by_name": policies = get_by_name(rc, req_json[service_type + "Info"]['policyId'], wildcards=True) elif osdf_config.core['policy_info'][service_type][ 'policy_fetch'] == "by_name_no_wildcards": policies = get_by_name(rc, req_json[service_type + "Info"]['policyId'], wildcards=False) else: policies = get_by_scope(rc, req_json, osdf_config.core, service_type) formatted_policies = [] for x in policies: if x[list(x.keys())[0]].get('properties') is None: raise BusinessException( "Properties not found for policy with name %s" % x[list(x.keys()[0])]) else: formatted_policies.append(x) return formatted_policies
def create_model_data(model_api): with app.app_context(): try: model_info = model_api['modelInfo'] model_id = model_info['modelId'] debug_log.debug("persisting model_api {}".format(model_id)) connection = get_db() cursor = connection.cursor(buffered=True) query = "SELECT model_id FROM optim_model_data WHERE model_id = %s" values = (model_id, ) cursor.execute(query, values) if cursor.fetchone() is None: query = "INSERT INTO optim_model_data (model_id, model_content, description, solver_type) VALUES " \ "(%s, %s, %s, %s)" values = (model_id, model_info['modelContent'], model_info.get('description'), model_info['solver']) cursor.execute(query, values) g.pg.commit() debug_log.debug( "A record successfully inserted for request_id: {}".format( model_id)) return retrieve_model_data(model_id) close_db() else: query = "UPDATE optim_model_data SET model_content = %s, description = %s, solver_type = %s where " \ "model_id = %s " values = (model_info['modelContent'], model_info.get('description'), model_info['solver'], model_id) cursor.execute(query, values) g.pg.commit() return retrieve_model_data(model_id) close_db() except Exception as err: error_log.error("error for request_id: {} - {}".format( model_id, traceback.format_exc())) close_db() raise BusinessException(err)
def delete_model_data(model_id): with app.app_context(): try: debug_log.debug( "deleting model data given model_id = {}".format(model_id)) connection = get_db() cursor = connection.cursor(buffered=True) query = "delete from optim_model_data WHERE model_id = %s" values = (model_id, ) cursor.execute(query, values) g.pg.commit() close_db() resp = { "statusMessage": "model data for modelId {} deleted".format(model_id) } return build_response(json.dumps(resp), 200) except Exception as err: error_log.error("error deleting model_id: {} - {}".format( model_id, traceback.format_exc())) close_db() raise BusinessException(err)
def request(req_info, demands, request_parameters, service_info, template_fields, osdf_config, flat_policies): config = osdf_config.deployment local_config = osdf_config.core uid, passwd = config['conductorUsername'], config['conductorPassword'] conductor_url = config['conductorUrl'] req_id = req_info["requestId"] transaction_id = req_info['transactionId'] headers = dict(transaction_id=transaction_id) placement_ver_enabled = config.get('placementVersioningEnabled', False) if placement_ver_enabled: cond_minor_version = config.get('conductorMinorVersion', None) if cond_minor_version is not None: x_minor_version = str(cond_minor_version) headers.update({'X-MinorVersion': x_minor_version}) debug_log.debug("Versions set in HTTP header to " "conductor: X-MinorVersion: {} ".format(x_minor_version)) max_retries = config.get('conductorMaxRetries', 30) ping_wait_time = config.get('conductorPingWaitTime', 60) rc = RestClient(userid=uid, passwd=passwd, method="GET", log_func=debug_log.debug, headers=headers) conductor_req_json_str = conductor_api_builder(req_info, demands, request_parameters, service_info, template_fields, flat_policies, local_config) conductor_req_json = json.loads(conductor_req_json_str) debug_log.debug("Sending first Conductor request for request_id {}".format(req_id)) resp, raw_resp = initial_request_to_conductor(rc, conductor_url, conductor_req_json) # Very crude way of keeping track of time. # We are not counting initial request time, first call back, or time for HTTP request total_time, ctr = 0, 2 client_timeout = req_info['timeout'] configured_timeout = max_retries * ping_wait_time max_timeout = min(client_timeout, configured_timeout) while True: # keep requesting conductor till we get a result or we run out of time if resp is not None: if resp["plans"][0].get("status") in ["error"]: raise RequestException(response=raw_resp, request=raw_resp.request) if resp["plans"][0].get("status") in ["done", "not found", "solved"]: return resp new_url = resp['plans'][0]['links'][0][0]['href'] # TODO(krishna): check why a list of lists if total_time >= max_timeout: raise BusinessException("Conductor could not provide a solution within {} seconds," "this transaction is timing out".format(max_timeout)) time.sleep(ping_wait_time) ctr += 1 debug_log.debug("Attempt number {} url {}; prior status={}" .format(ctr, new_url, resp['plans'][0]['status'])) total_time += ping_wait_time try: raw_resp = rc.request(new_url, raw_response=True) resp = raw_resp.json() except RequestException as e: debug_log.debug("Conductor attempt {} for request_id {} has failed because {}" .format(ctr, req_id, str(e)))
def test_handle_business_exception(self): e = BusinessException("Business Exception Description") resp = baseapp.handle_business_exception(e) assert resp.status_code == 400
def validate_request(request_json): OptimizationAPI(request_json).validate() if not is_valid_optim_request(request_json): raise BusinessException('Invalid optim request ') return True