def submit_model_data(self,
attachments=None,
gzip=False,
info_callback=None,
info_to_monitor=None):
"""Submits a job to the cloud service.
Args:
attachments: A list of attachments. Each attachement is a dict with
the following keys:
- 'name' : the name of the attachment
- 'data' : the data for the attachment
gzip: If ``True``, data is gzipped before sent over the network
info_callback: A call back to be called when some info are available.
That callback takes one parameter that is a dict containing
the info as they are available.
info_to_monitor: A set of information to monitor with info_callback.
Currently, can be ``jobid`` and ``progress``.
"""
self.__vars = None
self.timed_out = False
self.results.clear()
if not info_to_monitor:
info_to_monitor = {}
# check that url is valid
parts = urlparse(self.docloud_context.url)
if not parts.scheme:
raise DOcloudConnectorException(
"Malformed URL: '%s': No schema supplied." %
self.docloud_context.url)
proxies = self.docloud_context.proxies
try:
client = JobClient(self.docloud_context.url,
self.docloud_context.key,
proxies=proxies)
except TypeError:
# docloud client <= 1.0.172 do not have the proxes
warnings.warn(
"Using a docloud client that do not support warnings in init()",
UserWarning)
client = JobClient(self.docloud_context.url,
self.docloud_context.key)
self.log("client created")
if proxies:
self.log("proxies = %s" % proxies)
# prepare client
if self.docloud_context.log_requests:
client.rest_callback = \
lambda m, u, *a, **kw: self._rest_callback(m, u, *a, **kw)
client.verify = self.docloud_context.verify
client.timeout = self.docloud_context.get('timeout', None)
try:
try:
# Extract the list of attachment names
att_names = [a['name'] for a in attachments]
# create job
jobid = client.create_job(
attachments=att_names,
parameters=self.docloud_context.job_parameters)
self.log("job creation submitted, id is: {0!s}".format(jobid))
if info_callback and 'jobid' in info_to_monitor:
info_callback({'jobid': jobid})
except ConnectionError as c_e:
raise DOcloudConnectorException(
"Cannot connect to {0}, error: {1}".format(
self.docloud_context.url, str(c_e)))
try:
# now upload data
for a in attachments:
pos = 0
if 'data' in a:
att_data = {'data': a['data']}
elif 'file' in a:
att_data = {'file': a['file']}
pos = a['file'].tell()
elif 'filename' in a:
att_data = {'filename': a['filename']}
client.upload_job_attachment(jobid,
attid=a['name'],
**att_data)
self.log("Attachment: %s has been uploaded" % a['name'])
if self.docloud_context.debug_dump_dir:
target_dir = self.docloud_context.debug_dump_dir
if not os.path.exists(target_dir):
os.makedirs(target_dir)
self.log("Dumping input attachment %s to dir %s" %
(a['name'], target_dir))
with open(os.path.join(target_dir, a['name']),
"wb") as f:
if 'data' in 'a':
if isinstance(a['data'], bytes):
f.write(a['data'])
else:
f.write(a['data'].encode('utf-8'))
else:
a['file'].seek(pos)
f.write(a['file'])
# execute job
client.execute_job(jobid)
self.log("DOcplexcloud execute submitted has been started")
# get job execution status until it's processed or failed
timedout = False
try:
self._executionStatus = self.wait_for_completion(
client,
jobid,
info_callback=info_callback,
info_to_monitor=info_to_monitor)
except DOcloudInterruptedException:
timedout = True
self.log("docloud execution has finished")
# get job status. Do this before any time out handling
self.jobInfo = client.get_job(jobid)
if self.docloud_context.fire_last_progress and info_callback:
progress_data = self.map_job_info_to_progress_data(
self.jobInfo)
info_callback({'progress': progress_data})
if timedout:
self.timed_out = True
self.log("Solve timed out after {waittime} sec".format(
waittime=self.docloud_context.waittime))
return
# get solution => download all attachments
try:
for a in client.get_job_attachments(jobid):
if a['type'] == 'OUTPUT_ATTACHMENT':
name = a['name']
self.log("Downloading attachment '%s'" % name)
attachment_as_string = self._as_string(
client.download_job_attachment(jobid,
attid=name))
self.results[name] = attachment_as_string
if self.docloud_context.debug_dump_dir:
target_dir = self.docloud_context.debug_dump_dir
if not os.path.exists(target_dir):
os.makedirs(target_dir)
self.log("Dumping attachment %s to dir %s" %
(name, target_dir))
with open(os.path.join(target_dir, name),
"wb") as f:
f.write(
attachment_as_string.encode('utf-8'))
except DOcloudNotFoundError:
self.log("no solution in attachment")
self.log("docloud results have been received")
# on_solve_finished_cb
if self.docloud_context.on_solve_finished_cb:
self.docloud_context.on_solve_finished_cb(jobid=jobid,
client=client,
connector=self)
return
finally:
if self.docloud_context.delete_job:
deleted = client.delete_job(jobid)
self.log("delete status for job: {0!s} = {1!s}".format(
jobid, deleted))
finally:
client.close()
class Optimizer(object):
'''
Handles the actual optimization task.
Creates and executes a job builder for an optimization problem instance.
Encapsulates the DOCloud API.
This class is designed to facilitate multiple calls to the optimizer, such as would occur in a decomposition algorithm,
although it transparently supports single use as well.
In particular, the data can be factored into a constant data set that does not vary from run to run (represented by a JSON or .dat file)
and a variable piece that does vary (represented by a Collector object).
The optimization model can also be factored into two pieces, a best practice for large models and multi-models:
A data model that defines the tuples and tuple sets that will contain the input and output data.
An optimization model that defines the decision variables, decision expressions, objective function,
constraints, and pre- and post-processing data transformations.
Factoring either the data or the optimization model in this fashion is optional.
The problem instance is specified by the OPL model and input data received from the invoking (e.g. ColumnGeneration) instance.
Input and output data are realized as instances of OPLCollector, which in turn are specified by their respective schemas.
This class is completely independent of the specific optimization problem to be solved.
'''
def __init__(self, problemName, model=None, resultDataModel=None, credentials=None, *attachments):
'''
Constructs an Optimizer instance.
The instance requires an optimization model as a parameter.
You can also provide one or more data files as attachments, either in OPL .dat or in JSON format. This data does not
change from solve to solve. If you have input data that does change, you can provide it to the solve method as an OPLCollector object.
:param problemName: name of this optimization problem instance
:type problemName: String
:param model: an optimization model written in OPL
:type model: Model.Source object or String
:param resultDataModel: the application data model for the results of the optimization
:type resultDataModel: dict<String, StructType>
:param credentials: DOcplexcloud url and api key
:type credentials: {"url":String, "key":String}
:param attachments: URLs for files representing the data that does not vary from solve to solve
:type attachments: list<URL>
'''
self.name = problemName
self.model = model
self.resultDataModel = resultDataModel
self.attachData(attachments)
self.streamsRegistry = []
self.history = []
self.credentials = credentials
self.jobclient = JobClient(credentials["url"], credentials["key"]);
self.solveStatus = JobSolveStatus.UNKNOWN;
def getName(self):
"""
Returns the name of this problem
"""
return self.name
def setOPLModel(self, name, dotMods=None, modelText=None):
'''
Sets the OPL model.
This method can take any number of dotMod arguments, but
there are two common use cases:
First, the optimization model can be composed of two pieces:
A data model that defines the tuples and tuple sets that will contain the input and output data.
An optimization model that defines the decision variables, decision expressions, objective function,
constraints, and pre- and post-processing data transformations.
The two are concatenated, so they must be presented in that order.
If such a composite model is used, you do not need to import the data model into the optimization model using an OPL include statement.
Second, you do not have to use a separate data model, in which case a single dotMod must be provided
which encompasses both the data model and the optimization model.
@param name: the name assigned to this OPL model (should have the format of a file name with a .mod extension)
@type name: String
@param dotMods: URLs pointing to OPL .mod files, which will be concatenated in the order given
@type dotMods: List<URL>
@param modelText: the text of the OPL model, which will be concatenated in the order given
@type modelText: List<String>
@return this optimizer
@raise ValueError if a model has already been defined or if dotMods or modelText is empty
'''
if self.model is not None:
raise ValueError("model has already been set")
self.model = ModelSource(name=name, dotMods=dotMods, modelText=modelText)
return self
def setResultDataModel(self, resultDataModel):
'''
Sets the application data model for the results of the optimization
@param resultDataModel: the application data model for the results of the optimization
@type resultDataModel: dict<String, StructType>
'''
if self.resultDataModel is not None:
raise ValueError("results data model has already been defined")
self.resultDataModel = resultDataModel
return self
def attachData(self, attachments):
'''
Attaches one or more data files, either in OPL .dat or in JSON format. This data does not
change from solve to solve. If you have input data that does change, you can provide it as a Collector object.
@param attachments: files representing the data that does not vary from solve to solve
@type attachments: list<URL>
@return this optimizer
@raise ValueError if an item of the same name has already been attached
'''
self.attachments = {}
if attachments is not None:
for f in attachments:
fileName = os.path.splitext(os.path.basename(urlparse(f)))[0]
if fileName in self.attachments:
raise ValueError(fileName + " already attached")
self.attachments[fileName] = f
return self;
def solve(self, inputData=None, solutionId=""):
'''
Solves an optimization problem instance by calling the DOCloud solve service (Oaas).
Creates a new job request, incorporating any changes to the variable input data,
for a problem instance to be processed by the solve service.
Once the problem is solved, the results are mapped to an instance of an OPL Collector.
Note: this method will set a new destination for the JSON serialization of the input data.
@param inputData: the variable, solve-specific input data
@type inputData: OPLCollector
@param solutionId: an identifier for the solution, used in iterative algorithms (set to empty string if not needed)
@type solutionId: String
@return: a solution collector
'''
inputs = []
if self.model is None:
raise ValueError("A model attachment must be provided to the optimizer")
if self.model: # is not empty
stream = self.model.toStream()
inputs.append({"name": self.model.getName(), "file": stream})
self.streamsRegistry.append(stream)
if self.attachments: # is not empty
for f in self.attachments:
stream = urllib.FancyURLopener(self.attachments[f])
inputs.append({"name": f, "file": stream})
self.streamsRegistry.append(stream)
if inputData is not None:
outStream = cStringIO.StringIO()
inputData.setJsonDestination(outStream).toJSON()
inStream = cStringIO.StringIO(outStream.getvalue())
inputs.append({"name": inputData.getName() + ".json", "file": inStream})
self.streamsRegistry.extend([outStream, inStream])
response = self.jobclient.execute(
input=inputs,
output="results.json",
load_solution=True,
log="solver.log",
gzip=True,
waittime=300, # seconds
delete_on_completion=False)
self.jobid = response.jobid
status = self.jobclient.get_execution_status(self.jobid)
if status == JobExecutionStatus.PROCESSED:
results = cStringIO.StringIO(response.solution)
self.streamsRegistry.append(results)
self.solveStatus = response.job_info.get(
'solveStatus') # INFEASIBLE_SOLUTION or UNBOUNDED_SOLUTION or OPTIMAL_SOLUTION or...
solution = (OPLCollector(self.getName() + "Result" + solutionId, self.resultDataModel)).setJsonSource(
results).fromJSON()
self.history.append(solution)
elif status == JobExecutionStatus.FAILED:
# get failure message if defined
message = ""
if (response.getJob().getFailureInfo() != None):
message = response.getJob().getFailureInfo().getMessage()
print("Failed " + message)
else:
print("Job Status: " + status)
for s in self.streamsRegistry:
s.close();
self.jobclient.delete_job(self.jobid);
return solution
def getSolveStatus(self):
"""
@return the solve status as a string
Attributes:
UNKNOWN: The algorithm has no information about the solution.
FEASIBLE_SOLUTION: The algorithm found a feasible solution.
OPTIMAL_SOLUTION: The algorithm found an optimal solution.
INFEASIBLE_SOLUTION: The algorithm proved that the model is infeasible.
UNBOUNDED_SOLUTION: The algorithm proved the model unbounded.
INFEASIBLE_OR_UNBOUNDED_SOLUTION: The model is infeasible or unbounded.
"""
return self.solveStatus
