def _create_solver_agent(self, aname):
""" Create a new solver agent from its name.
Args:
name: Name of the agent
Returns:
Solver agent instance
Raises:
CpoException: Agent creation error
"""
# Get agent context
sctx = self.context.solver.get(aname)
if not isinstance(sctx, Context):
raise CpoException("Unknown solving agent '" + aname + "'. Check config.context.solver.agent parameter.")
if sctx.is_log_enabled(3):
sctx.log(3, "Context for solving agent '", aname, "':")
sctx.write(out=sctx.get_log_output())
cpath = sctx.class_name
if cpath is None:
raise CpoException("Solving agent '" + aname + "' context should contain an attribute 'class_name'")
# Retrieve solver agent class
try:
sclass = utils.get_module_element_from_path(cpath)
except Exception as e:
raise CpoException("Unable to retrieve solver agent class '{}': {}".format(cpath, e))
if not inspect.isclass(sclass):
raise CpoException("Solver agent '{}' is not a class.".format(cpath))
if not issubclass(sclass, CpoSolverAgent):
raise CpoException("Solver agent class '{}' should extend CpoSolverAgent.".format(cpath))
# Create agent instance
agent = sclass(self, sctx.params, sctx)
return agent
def _add_listener_from_class(self, lstnr):
""" Add a solver listener from its class (instance is created).
Args:
lstnr: Solver listener class, or string identifying the class
"""
if is_string(lstnr):
# Get listener class from string
try:
lclass = utils.get_module_element_from_path(lstnr)
except Exception as e:
raise CpoException("Unable to retrieve solver listener class '{}': {}".format(lstnr, e))
if not inspect.isclass(lclass):
raise CpoException("Solver listener '{}' is not a class.".format(lstnr))
if not issubclass(lclass, CpoSolverListener):
raise CpoException("Solver listener class '{}' should extend CpoSolverListener.".format(lstnr))
else:
# Listener is assumed to directly be a class
lclass = lstnr
if not inspect.isclass(lclass):
raise CpoException("Solver listener '{}' is not a class.".format(lclass))
if not issubclass(lclass, CpoSolverListener):
raise CpoException("Solver listener class '{}' should extend CpoSolverListener.".format(lclass))
# Add listener
self.add_listener(lclass())
def _check_status(self, ests):
""" Throws an exception if solver status is not the expected one
Args:
ests: Expected status, or list of expected statuses
Raise:
CpoException if solver is not in the right status
"""
if self.status != ests:
raise CpoException("Unexpected solver status. Should be '{}' instead of '{}'".format(ests, self.status))
def _get_effective_context(**kwargs):
""" Build a effective context from a variable list of arguments that may specify changes to default.
Args:
context (optional): Source context, if not default.
params (optional): Solving parameters (CpoParameters) that overwrite those in the solving context
(others) (optional): All other context parameters that can be changed.
Returns:
Updated (cloned) context
"""
# If 'url' and 'key' are defined, force agent to be docloud
if ('agent' not in kwargs) and not ENVIRONMENT_PRESENT:
url = kwargs.get('url')
key = kwargs.get('key')
if url and key and is_string(url) and is_string(key) and url.startswith('http'):
kwargs['agent'] = 'docloud'
# Determine source context
ctx = kwargs.get('context')
if (ctx is None) or (ctx in DEFAULT_VALUES):
ctx = context
ctx = ctx.clone()
# print("\n*** Source context");
# ctx.print_context()
# First set parameters if given
prms = kwargs.get('params')
if prms is not None:
ctx.params.add(prms)
# Process other changes
rplist = [] # List of replacements to be done in solving parameters
for k, v in kwargs.items():
if (k != 'context') and (k != 'params') and (v not in DEFAULT_VALUES):
rp = ctx.search_and_replace_attribute(k, v)
# If not found, set in solving parameters
if (rp is None):
rplist.append((k, v))
# Replace or set remaining fields in parameters
if rplist:
params = ctx.params
chkparams = not ctx.solver.enable_undocumented_params
if isinstance(params, CpoParameters):
for k, v in rplist:
if chkparams and not k in ALL_PARAMETER_NAMES:
raise CpoException("CPO solver does not accept a parameter named '{}'".format(k))
setattr(params, k, v)
# Return
# print("\n*** Result context");
# ctx.print_context()
return ctx
def _get_solver_agent(self):
""" Get the solver agent instance that is used to solve the model.
Returns:
Solver agent instance
Raises:
CpoException: Agent creation error
"""
# Determine selectable agent(s)
sctx = self.context.solver
alist = sctx.agent
if alist is None:
alist = 'docloud'
elif not (is_string(alist) or is_array(alist)):
raise CpoException("Agent identifier in config.context.solver.agent should be a string or a list of strings.")
# Create agent
if is_string(alist):
aname = alist
agent = self._create_solver_agent(alist)
else:
# Search first available agent in the list
agent = None
aname = None
errors = []
for aname in alist:
try:
agent = self._create_solver_agent(aname)
break
except Exception as e:
errors.append((aname, str(e)))
# Agent not found
errstr = ', '.join(a + ": " + str(e) for (a, e) in errors)
raise CpoException("Agent creation error: " + errstr)
# Log solver agent
sctx.log(1, "Solve model '", self.model.get_name(), "' with agent '", aname, "'")
agent.process_infos[CpoProcessInfos.SOLVER_AGENT] = aname
return agent
def __init__(self, solver, params, context):
""" Create a new solver using DOcplexcloud web service
Args:
solver: Parent solver
params: Solving parameters
context: DOcplexcloud Solver context
Raises:
CpoException if jar file does not exists
"""
if (context.key is None) or (' ' in context.key):
raise CpoException("Your DOcplexcloud key has not been set")
super(CpoSolverDocloud, self).__init__(solver, params, context)
def __init__(self, solver, params, context):
""" Create a new solver using DOcplexcloud web service
Args:
solver: Parent solver
params: Solving parameters
context: DOcplexcloud Solver context
Raises:
CpoException if jar file does not exists
"""
warnings.warn(
'Solve using \'docloud\' agent is deprecated. Consider submitting your model to DOcplexcloud. See https://ibm.biz/BdYhhK',
DeprecationWarning)
if (context.key is None) or (' ' in context.key):
raise CpoException("Your DOcplexcloud key has not been set")
super(CpoSolverDocloud, self).__init__(solver, params, context)
def _create_solver_agent(self, aname):
""" Create a new solver agent from its name.
Args:
name: Name of the agent
Returns:
Solver agent instance
Raises:
CpoException: Agent creation error
"""
# Get agent context
sctx = self.context.solver.get(aname)
if not isinstance(sctx, Context):
raise CpoException("Unknown solving agent '" + aname + "'. Check config.context.solver.agent parameter.")
if sctx.is_log_enabled(3):
sctx.log(3, "Context for solving agent '", aname, "':")
sctx.print_context(out=sctx.get_log_output())
cpath = sctx.class_name
if cpath is None:
raise CpoException("Solving agent '" + aname + "' context does not contain attribute 'class_name'")
# Split class name
pnx = cpath.rfind('.')
if pnx < 0:
raise CpoException("Invalid class name '" + cpath + "' for solving agent '" + aname + "'. Should be <package>.<module>.<class>.")
mname = cpath[:pnx]
cname = cpath[pnx + 1:]
# Load module
try:
module = importlib.import_module(mname)
except Exception as e:
raise CpoException("Module '" + mname + "' import error: " + str(e))
# Create and check class
sclass = getattr(module, cname, None)
if sclass is None:
raise CpoException("Module '" + mname + "' does not contain a class '" + cname + "'")
if not inspect.isclass(sclass):
raise CpoException("Agent class '" + cpath + "' is not a class.")
if not issubclass(sclass, CpoSolverAgent):
raise CpoException("Solver agent class '" + cpath + "' does not extend CpoSolverAgent.")
# Create agent instance
agent = sclass(self, sctx.params, sctx)
return agent
def _get_solver_agent_class(aname, sctx):
""" Get a solver agent class from its name
Args:
aname: Solver agent name
sctx: Candidate solver context
Returns:
Solver agent class
"""
# Check for solver agent context
if not isinstance(sctx, Context):
raise CpoException("Unknown solving agent '" + aname + "'. Check config.context.solver.agent parameter.")
cpath = sctx.class_name
if cpath is None:
raise CpoException("Solving agent '" + aname + "' context does not contain attribute 'class_name'")
# Split class name
pnx = cpath.rfind('.')
if pnx < 0:
raise CpoException("Invalid class name '" + cpath + "' for solving agent '" + aname + "'. Should be <package>.<module>.<class>.")
mname = cpath[:pnx]
cname = cpath[pnx + 1:]
# Load module
try:
module = importlib.import_module(mname)
except Exception as e:
raise CpoException("Module '" + mname + "' import error: " + str(e))
# Create and check class
sclass = getattr(module, cname, None)
if sclass is None:
raise CpoException("Module '" + mname + "' does not contain a class '" + cname + "'")
if not inspect.isclass(sclass):
raise CpoException("Agent class '" + cpath + "' is not a class.")
if not issubclass(sclass, CpoSolverAgent):
raise CpoException("Solver agent class '" + cpath + "' does not extend CpoSolverAgent.")
# Return
return sclass
def _submit_job(self, cmd, rclass):
""" Submit a solving job with given command.
According to the value of the context parameter 'verbose', the following information is logged
if the log output is set:
* 1: Total time spent to solve the model
* 2: Calls to DOcplexcloud solving
* 3: Detailed DOcplexcloud job information
* 4: REST requests and response codes
Args:
cmd: Solving command, in (Solve, RefineConflict, Propagate)
rclass: Result object class
Returns:
Model solve result (object of class CpoSolveResult)
Raises:
CpoException if error occurs
"""
# Create DOcplexcloud client
ctx = self.context
client = JobClient(ctx)
# Convert model into CPO format
cpostr = self._get_cpo_model_string()
# Encode model
stime = time.time()
cpostr = cpostr.encode('utf-8')
self.process_infos[
CpoProcessInfos.TOTAL_UTF8_ENCODE_TIME] = time.time() - stime
# Solve model and retrieve solution
name = self.model.get_name()
if is_number(ctx.params.TimeLimit):
maxwait = ctx.params.TimeLimit + ctx.request_timeout + ctx.result_wait_extra_time
else:
maxwait = 0
try:
# Create job and start execution
stime = time.time()
client.create_cpo_job(name, cpostr, cmd)
self.process_infos[
CpoProcessInfos.TOTAL_DATA_SEND_TIME] = time.time() - stime
# Start execution
client.execute_job()
# Wait job termination
lgntf = None
if self.log_enabled:
lgntf = (
lambda recs: self._add_log_data(''.join(r['message'] + "\n"
for r in recs)))
client.wait_job_termination(maxwait=maxwait, lognotif=lgntf)
jinfo = client.get_info()
# Trace response if required
if ctx.is_log_enabled(3):
ctx.log(3, "Job info:")
for k in jinfo.keys():
ctx.log(3, k, " : ", jinfo[k])
# Check failure
fail = jinfo.get('failure', None)
if fail is not None:
raise CpoException(fail.get('message', "Unknown failure"))
# Get solve status
ssts = _get_solve_status(jinfo)
if (cmd == "Solve") and (ssts not in _STATUS_WITH_RESULT):
jsol = None
else:
# Get solution
try:
stime = time.time()
jsol = client.get_attachment("solution.json")
self.process_infos[
CpoProcessInfos.
RESULT_RECEIVE_TIME] = time.time() - stime
self.process_infos[CpoProcessInfos.RESULT_DATA_SIZE] = len(
jsol)
stime = time.time()
jsol = jsol.decode('utf-8')
self.process_infos[
CpoProcessInfos.
TOTAL_UTF8_DECODE_TIME] = time.time() - stime
except Exception as e:
raise CpoException("Model solution access error: " +
str(e))
# Create solution structure
msol = self._create_result_object(rclass, jsol)
if not jsol:
msol.solve_status = ssts
sinfos = msol.solver_infos
# Add solve details if no json solution
detls = jinfo.get('details', None)
if detls is not None:
# Retrieve common infos
for k in detls:
if not (('.' in k) or ('_' in k)):
sinfos[k] = _value_of(detls[k])
# Retrieve special hard-coded values
if isinstance(msol, CpoSolveResult
) and msol.solution.objective_values is None:
oval = detls.get('PROGRESS_CURRENT_OBJECTIVE', None)
if oval is not None:
msol.solution.objective_values = [
float(v) for v in oval.split(";")
]
sinfos.setdefault(
CpoSolverInfos.NUMBER_OF_CONSTRAINTS,
int(detls.get('MODEL_DETAIL_CONSTRAINTS', 0)))
sinfos.setdefault(
CpoSolverInfos.NUMBER_OF_INTEGER_VARIABLES,
int(detls.get('MODEL_DETAIL_INTEGER_VARS', 0)))
sinfos.setdefault(
CpoSolverInfos.NUMBER_OF_INTERVAL_VARIABLES,
int(detls.get('MODEL_DETAIL_INTERVAL_VARS', 0)))
sinfos.setdefault(
CpoSolverInfos.NUMBER_OF_SEQUENCE_VARIABLES,
int(detls.get('MODEL_DETAIL_SEQUENCE_VARS', 0)))
# Add solve time
if CpoSolverInfos.SOLVE_TIME not in sinfos:
sinfos[CpoSolverInfos.SOLVE_TIME] = (
float(jinfo.get('endedAt', 0)) -
float(jinfo.get('startedAt', 0))) / 1000
finally:
# Delete job
if ctx.clean_job_after_solve:
client.clean_job()
# Return
return msol
def __init__(self, solver, params, context):
""" Create a new solver that solves locally with CP Optimizer Interactive.
Args:
solver: Parent solver
params: Solving parameters
context: Solver context
Raises:
CpoException if proxy executable does not exists
"""
# Call super
self.process = None
self.active = True
super(CpoSolverLocal, self).__init__(solver, params, context)
# Check if executable file exists
if context.execfile is None:
raise CpoException(
"Executable file should be given in 'execfile' context attribute."
)
if not is_string(context.execfile):
raise CpoException(
"Executable file should be given in 'execfile' as a string.")
#if not os.path.isfile(context.execfile):
# raise CpoException("Executable file '" + str(context.execfile) + "' does not exists")
# Create solving process
cmd = [context.execfile]
if context.parameters is not None:
cmd.extend(context.parameters)
context.log(2, "Solver exec command: '", ' '.join(cmd), "'")
try:
self.process = subprocess.Popen(cmd,
stdin=subprocess.PIPE,
stderr=subprocess.STDOUT,
stdout=subprocess.PIPE,
universal_newlines=False)
except:
raise CpoException(
"Can not execute command '{}'. Please check availability of required executable file."
.format(' '.join(cmd)))
self.pout = self.process.stdin
self.pin = self.process.stdout
# Read initial version info from process
self.version_info = None
timer = threading.Timer(
1, lambda: self.process.kill()
if self.version_info is None else None)
timer.start()
evt, data = self._read_message()
timer.cancel()
if evt != EVT_VERSION_INFO:
raise LocalSolverException(
"Unexpected event {} received instead of version info event {}."
.format(evt, EVT_VERSION_INFO))
self.version_info = verinf = json.loads(data)
self.available_command = self.version_info['AvailableCommands']
# Normalize information
verinf['AgentModule'] = __name__
iver = verinf.get('AngelVersion')
if iver:
verinf['InterfaceVersion'] = iver
context.log(3, "Local solver info: '", verinf, "'")
# Check solver version if any
sver = self.version_info.get('SolverVersion')
mver = solver.get_model_format_version()
if sver and mver and compare_natural(mver, sver) > 0:
raise LocalSolverException(
"Solver version {} is lower than model format version {}.".
format(sver, mver))
# Send CPO model to process
cpostr = self._get_cpo_model_string()
self._write_message(CMD_SET_CPO_MODEL, cpostr)
self._wait_json_result(EVT_SUCCESS) # JSON stored
context.log(3, "Model sent.")
# Initialize CPO callback setting
self.callback_added = False
def __init__(self, solver, params, context):
""" Create a new solver that solves locally with CP Optimizer Interactive.
Args:
solver: Parent solver
params: Solving parameters
context: Solver context
Raises:
CpoException if proxy executable does not exists
"""
# Call super
self.process = None
self.active = True
self.timeout_kill = False
super(CpoSolverLocal, self).__init__(solver, params, context)
# Check if executable file exists
xfile = context.execfile
if xfile is None or not is_string(xfile):
raise CpoException(
"Executable file should be given in 'execfile' context attribute."
)
if not os.path.isfile(xfile):
raise CpoException(
"Executable file '{}' does not exists".format(xfile))
if not is_exe_file(xfile):
raise CpoException(
"Executable file '{}' is not executable".format(xfile))
# Create solving process
cmd = [context.execfile]
if context.parameters is not None:
cmd.extend(context.parameters)
context.log(2, "Solver exec command: '", ' '.join(cmd), "'")
try:
self.process = subprocess.Popen(cmd,
stdin=subprocess.PIPE,
stderr=subprocess.STDOUT,
stdout=subprocess.PIPE,
universal_newlines=False)
except:
raise CpoException(
"Can not execute command '{}'. Please check availability of required executable file."
.format(' '.join(cmd)))
self.pout = self.process.stdin
self.pin = self.process.stdout
self.out_lock = threading.Lock()
# Read initial version info from process
self.version_info = None
timeout = context.process_start_timeout
timer = threading.Timer(timeout, self._process_start_timeout)
timer.start()
try:
evt, data = self._read_message()
except Exception as e:
if self.timeout_kill:
raise CpoSolverException(
"Solver process was too long to start and respond ({} seconds). Process has been killed."
.format(timeout))
raise CpoSolverException(
"Solver sub-process start failure: {}".format(e))
timer.cancel()
# Check received message
if evt != EVT_VERSION_INFO:
raise CpoSolverException(
"Unexpected event {} received instead of version info event {}."
.format(evt, EVT_VERSION_INFO))
self.version_info = verinf = json.loads(data)
self.available_commands = self.version_info['AvailableCommands']
# Normalize information
verinf['AgentModule'] = __name__
context.log(3, "Local solver info: '", verinf, "'")
# Transfer infos in process info
for x in ('ProxyVersion', 'AngelVersion', 'SourceDate',
'SolverVersion'):
self.process_infos[x] = self.version_info.get(x)
# Check solver version if any
sver = self.version_info.get('SolverVersion')
mver = solver.get_model_format_version()
if sver and mver and compare_natural(mver, sver) > 0:
raise CpoSolverException(
"Solver version {} is lower than model format version {}.".
format(sver, mver))
def __init__(self, solver, params, context):
""" Create a new solver that solves locally with CP Optimizer Interactive.
Args:
solver: Parent solver
params: Solving parameters
context: Proxy solver context
Raises:
CpoException if proxy executable does not exists
"""
# Call super
self.process = None
self.active = True
super(CpoSolverLocal, self).__init__(solver, params, context)
# Check if executable file exists
if context.execfile is None:
raise CpoException("Executable file should be given in 'execfile' context attribute.")
if not is_string(context.execfile):
raise CpoException("Executable file should be given in 'execfile' as a string.")
#if not os.path.isfile(context.execfile):
# raise CpoException("Executable file '" + str(context.execfile) + "' does not exists")
# Create solving process
cmd = [context.execfile]
if context.parameters is not None:
cmd.extend(context.parameters)
context.log(2, "Solver exec command: '", ' '.join(cmd), "'")
try:
self.process = subprocess.Popen(cmd, stdin=subprocess.PIPE, stderr=subprocess.STDOUT, stdout=subprocess.PIPE, universal_newlines=False)
except:
raise CpoException("Can not execute command '{}'. Please check availability of required executable file.".format(' '.join(cmd)))
self.pout = self.process.stdin
self.pin = self.process.stdout
# Read initial version info from process
self.version_info = None
timer = threading.Timer(1, lambda: self.process.kill() if self.version_info is None else None)
timer.start()
evt, data = self._read_message()
timer.cancel()
if evt != EVT_VERSION_INFO:
raise LocalSolverException("Unexpected event {} received instead of version info event {}.".format(evt, EVT_VERSION_INFO))
self.version_info = json.loads(data.decode('utf-8'))
self.available_command = self.version_info['AvailableCommands']
context.log(3, "Local solver info: '", self.version_info, "'")
# Convert model into CPO format
cpostr = self._get_cpo_model_string()
# Encode model
stime = time.time()
cpostr = cpostr.encode('utf-8')
self.process_infos[CpoProcessInfos.MODEL_ENCODE_TIME] = time.time() - stime
# Send CPO model to process
stime = time.time()
self._write_message(CMD_SET_CPO_MODEL, cpostr)
self.process_infos[CpoProcessInfos.MODEL_SEND_TIME] = time.time() - stime
context.log(3, "Model sent.")
self._wait_json_result(EVT_SUCCESS) # JSON stored