class ControllerMultiJob:
""" This controller submit single job asynchronously. """
def __init__(self, url, api_key, concurrent_jobs):
""" Creates a new controller to submit job asynchronously.
Args:
url: The DOcloud url.
api_key: The DOcloud api key.
concurrent_jobs: The number of concurrently submitted jobs.
"""
self.nb_threads = concurrent_jobs
self.client = JobClient(url, api_key)
# This is the opl model file
self.mod_file = "models/truck.mod"
# The executor
self.executor = ThreadPoolExecutor(self.nb_threads)
def shutdown(self):
self.executor.shutdown()
def submitJob(self, pb, responses, latch):
"""Asynchronously submits a single job.
Once the job has finished running, the latch is decreased.
Args:
pb: The ``Problem`` to submit and run.
responses: A list of responses where the response for this ``pb`` is added.
latch: The latch to decrease once the ``Problem`` has been solved.
"""
def submitAndCountdown(pb, responses, latch):
# Encodes the problem using the specified encoder (which extends
# json.JSONEncoder)
data = json.dumps(pb, cls=ProblemEncoder).encode('utf-8')
print("Running %s" % pb.problem_id)
resp = self.client.execute(
input=[{
'name': "truck.mod",
'filename': self.mod_file
}, {
'name': "truck.json",
'data': data
}],
gzip=True,
load_solution=True,
delete_on_completion=True,
parameters={'oaas.client.problem.id': pb.problem_id})
responses.append(resp)
latch.count_down()
self.executor.submit(submitAndCountdown, pb, responses, latch)
def run_model():
url = 'https://api-oaas.docloud.ibmcloud.com/job_manager/rest/v1/'
key = 'api_fc2d8028-3ebc-4a3a-8664-b4018b1c05a8'
client = JobClient(url=url, api_key=key)
resp = client.execute(input=[
'workshop_model.py', '../input/best_submission.csv',
'../input/family_data.csv'
],
output='solution.json',
load_solution=True,
log='logs.txt')
class ControllerMultiJob:
""" This controller submit single job asynchronously. """
def __init__(self, url, api_key, concurrent_jobs):
""" Creates a new controller to submit job asynchronously.
Args:
url: The DOcloud url.
api_key: The DOcloud api key.
concurrent_jobs: The number of concurrently submitted jobs.
"""
self.nb_threads = concurrent_jobs
self.client = JobClient(url, api_key)
# This is the opl model file
self.mod_file = "models/truck.mod"
# The executor
self.executor = ThreadPoolExecutor(self.nb_threads)
def shutdown(self):
self.executor.shutdown()
def submitJob(self, pb, responses, latch):
"""Asynchronously submits a single job.
Once the job has finished running, the latch is decreased.
Args:
pb: The ``Problem`` to submit and run.
responses: A list of responses where the response for this ``pb`` is added.
latch: The latch to decrease once the ``Problem`` has been solved.
"""
def submitAndCountdown(pb, responses, latch):
# Encodes the problem using the specified encoder (which extends
# json.JSONEncoder)
data = json.dumps(pb, cls=ProblemEncoder).encode('utf-8')
print("Running %s" % pb.problem_id)
resp = self.client.execute(input=[{'name': "truck.mod",
'filename': self.mod_file},
{'name': "truck.json",
'data': data}],
gzip=True,
load_solution=True,
delete_on_completion=True,
parameters={'oaas.client.problem.id': pb.problem_id})
responses.append(resp)
latch.count_down()
self.executor.submit(submitAndCountdown, pb, responses, latch)
class KestrelGamsClient:
def __init__(self,argv):
self.time1 = time.time()
self.argv=argv
self.serverProtocol="https"
self.serverHost="neos-server.org"
self.doCloudUrl=None
self.doCloudKey=None
self.doCloudClient=None
self.doCloudPrmFile=None
self.serverPort=3333
self.solverName=None
self.jobNumber=None
self.password=None
self.priority="long"
self.socket_timeout=0
self.authUsername=None
self.authUserPassword=None
self.gmo = None
self.gev = None
# action-parameter is outdated
'''
if len(self.argv) >= 3:
self.cntrfile = self.argv[2]
self.action = self.argv[1].lower()
if self.action not in ['kill','retrieve','submit','solve']:
self.Usage()
else:
self.Usage()
'''
if len(self.argv) >= 2:
self.cntrfile = self.argv[1]
self.action = 'solve'
else:
self.Usage()
def Usage(self):
sys.stderr.write("\n--- Kestrel fatal error: usage\n")
sys.stderr.write(" gamske_ux.out <cntrfile>\n")
sys.exit(1)
def Fatal(self, str):
sys.stderr.write("\n--- Kestrel fatal error: %s\n\n" % str)
sys.exit(1)
def Error(self, str):
if self.logopt in [1,3,4]:
# Write the message to standard output
sys.stdout.write("\n--- Kestrel error: %s\n\n" % str)
if self.logopt in [2,4]:
# Append the error message to the logfile indicated
try:
f = open(self.logfilename,'a')
f.write("\n--- Kestrel error: %s\n\n" % str)
f.close()
except IOError as e:
self.Fatal("Could not append to log file %s" % self.logfilename)
try:
f = open(self.statfilename,'a')
f.write("=1\n\n--- Kestrel error: %s\n\n=2\n" % str)
f.close()
except IOError as e:
self.Fatal("Could not append to status file %s\n" % self.statfilename)
sys.exit(0)
def getDefaultEmail(self):
if 'NEOS_EMAIL' in os.environ:
return os.environ['NEOS_EMAIL']
return None
def parseControlFile(self):
"""
This function does the following with the cntr file
line 13:
extract isAscii, useOptions
line 18:
matrix file, save and change to gamsmatr.scr
line 19:
instruction file; save and change to gamsinst.scr
line 20:
set options file to 'kestrel.opt'
line 21:
status file; save and change to gamsstat.scr
line 22:
solution file; save and change to gamssolu.scr
line 23:
log file; save and remove absolute path
line 24:
dictionary file; save and change to gamsdict.scr
line 25:
set to '2' to write to log file
line 28-30:
set working,system,scratch directories to '.'
line 33,34,35:
remove license
line 37:
set parameter file
line 38:
read #models #solvers
ignore next #models + 2*#solvers with (SOLVER # # 0 ..) +
3*#solvers with (SOLVER # # 1 ...) lines
next two lines are more license (remove them)
set directories of remaining paths to current directory
(.scr, .so, sbbinfo.)
change the scratch file extension to 'scr'
"""
try:
f = open(self.cntrfile,'r')
lines = f.readlines()
f.close()
except IOError as e:
self.Fatal("Could not open control file %s" % self.cntrfile)
# extract control version number
self.cntver = 0
m = re.match(r'(\d+)',lines[0])
if m and m.groups():
self.cntver = int(m.groups()[0])
self.modeltype = int(lines[1].split()[0])
#if self.cntver != 41 and self.cntver != 42:
# self.Fatal("GAMS 22.x required")
if self.cntver not in [41, 42, 44, 46, 47, 48, 49, 50]:
self.Fatal("GAMS cntr-file version 41, 42, 44, 46, 47, 48, 49, 50 required")
# extract isAscii, useOptions
m = re.match(r'(\d+)\s+(\d+)',lines[12])
if m and m.groups():
self.isAscii=m.groups()[0]
self.useOptions = int(m.groups()[1])
else:
self.Fatal("Line 13 of the control file is incorrect")
# is this is an MPSGE model?
self.isMPSGE = int(lines[15].split()[0])
# get the matrix and instruction scratch files and patch
self.matrfilename = lines[17].strip()
lines[17] = "gamsmatr.scr\n"
self.instfilename = lines[18].strip()
lines[18] = "gamsinst.scr\n"
# patch option file name; always use kestrel.opt
self.optfilename = ""
m = re.match(r'(.*)kestrel.*\.(.*)',lines[19])
if m and m.groups():
self.optfilename = m.groups()[0] + "kestrel." + m.groups()[1]
lines[19] = "kestrel.opt\n"
# get the status and solution scratch files and patch
self.statfilename = lines[20].strip()
lines[20] = "gamsstat.scr\n"
self.solufilename = lines[21].strip()
lines[21] = "gamssolu.scr\n"
# get the log filename and patch
self.logfilename = lines[22].strip()
lines[22] = "gamslog.scr\n"
# get the dictionary filename and patch
self.dictfilename = lines[23].strip()
lines[23] = "gamsdict.scr\n"
# get the logfile option, then make output written to logfile
m = re.match(r'(\d+)',lines[24])
if m and m.groups():
self.logopt = int(m.groups()[0])
lines[24]="2\n"
# set working, system, and scratch directories
self.scrdir = lines[29].strip()
lines[27] = lines[28] = lines[29] = '.\n'
# remove first part of license
lines[32] = lines[33] = lines[34] = "\n"
# patch parameter file
lines[36] = "gmsprmun.scr"
# downgrade the cntr-file version 50 to 49 // No change required since this was in the license section which does not get copied
if self.cntver == 50:
# 50 -> 49
lines[0] = "49\n"
self.cntver = 49
# downgrade the cntr-file version 49 to 48 // No change required since this was in the license section which does not get copied
if self.cntver == 49:
# 49 -> 48
lines[0] = "48\n"
self.cntver = 48
# downgrade the cntr-file version 48 to 47
if self.cntver == 48:
# 48 -> 47
lines[0] = "47\n"
self.cntver = 47
# remove last two numbers of this line
lines[13] = lines[13].rpartition(' ')[0] + "\n"
lines[13] = lines[13].rpartition(' ')[0] + "\n"
# downgrade the cntr-file version 47 to 46
if self.cntver == 47:
# 47 -> 46
lines[0] = "46\n"
self.cntver = 46
# remove line with file name
lines = lines[:-2]
lines.append("")
# downgrade the cntr-file version 46 to 42
# no support for threads, external funclib and scensolver
if self.cntver == 46:
# 46 -> 42
lines[0] = "42\n"
# remove last number of this line
lines[2] = lines[2].rpartition(' ')[0] + "\n"
# remove threads-option
lines[13] = lines[13].rpartition(' ')[0] + "\n"
# remove last two lines
lines = lines[:-2]
# treat the cntr-file now like a version 42 one
self.cntver = 42
# downgrade the cntr-file version 44 to 42
elif self.cntver == 44:
# 44 -> 42
lines[0] = "42\n"
# remove threads-option
lines[13] = lines[13].rpartition(' ')[0] + "\n"
# remove last line
lines = lines[:-1]
# treat the cntr-file now like a version 42 one
self.cntver = 42
# ignore solver section and patch rest based on version number
if self.cntver == 41:
# remove second part of license
lines[-11] = lines[-10] = "\n"
# make everything in local directory
lines[-9] = 'model.scr\n'
lines[-4] = 'model.so\n'
lines[-3] = 'sbbinfo.scr\n'
lines[-2] = 'gamscntr.scr\n'
lines[-1] = './\n'
# set scratch file extension
self.scrext = "scr"
# get the entire control file name
self.cntr = "".join(lines[:37]) + "".join(lines[-11:])
elif self.cntver == 42:
# remove second part of license
lines[-13] = lines[-12] = "\n"
# make everything in local directory
lines[-11] = 'model.scr\n'
lines[-6] = 'model.so\n'
lines[-5] = 'sbbinfo.scr\n'
lines[-4] = 'gamscntr.scr\n'
lines[-3] = './\n'
# patch scratch file extension
self.scrext = lines[-2].strip()
lines[-2] = 'scr\n'
# get the entire control file name
self.cntr = "".join(lines[:37]) + "".join(lines[-13:])
def writeErrorOutputFiles(self):
"""
This writes solution and status files returned when an error occurs.
"""
try:
f = open(self.statfilename,"w")
f.write("""=0 Kestrel\n""")
f.close()
except IOError as e:
self.Error("Could not initialize status file %s\n" % self.statfilename)
try:
f = open(self.solufilename,"w")
f.write(""" 1 6.0000000000000000E+00
2 1.3000000000000000E+01
3 0.0000000000000000E+00
4 0.0
5 0.0000000000000000E+00
6 0.0
7 0.0
8 0.0
0 0.0\n""")
f.close()
except IOError as e:
self.Error("Could not open solution file %s\n" % self.solufilename)
def writeLog(self, text):
if self.logopt in [1,3,4]:
sys.stdout.write(text)
if self.logopt in [2,4]:
try:
f = open(self.logfilename,'a')
f.write(text)
f.close()
except IOError as e:
self.Fatal("Could not append to log file %s" % self.logfilename)
def parseOptionsFile(self):
if (self.useOptions == 0):
# raise KestrelSolverException("No options file indicated\n",self.kestrelGamsSolvers)
self.solverName = solverMap[self.modeltype]
elif os.access(self.optfilename,os.R_OK):
optfile = open(self.optfilename,'r')
self.writeLog("Reading parameter(s) from \"" + self.optfilename + "\"\n")
for line in optfile:
m = re.match(r'neos_user_password[\s=]+(\S+)',line)
if m:
self.writeLog(">> neos_user_password ******")
else:
self.writeLog(">> " + line)
m = re.match(r'kestrel_priority[\s=]+(\S+)',line)
if m:
value = m.groups()[0]
if value.lower()=="short":
self.priority = "short"
m = re.match(r'kestrel_solver[\s=]+(\S+)',line)
if m:
self.solverName = m.groups()[0]
m = re.match(r'neos_server[\s=]+(\S+)://(\S+):(\d+)',line)
if m:
self.serverProtocol = m.groups()[0]
self.serverHost = m.groups()[1]
self.serverPort = m.groups()[2]
m = re.match(r'neos_username[\s=]+(\S+)',line)
if m:
self.authUsername = m.groups()[0]
m = re.match(r'neos_user_password[\s=]+(\S+)',line)
if m:
self.authUserPassword = m.groups()[0]
elif re.match(r'neos_server[\s=]+(\S+)://(\S+)',line):
m = re.match(r'neos_server[\s=]+(\S+)://(\S+)',line)
self.serverProtocol = m.groups()[0]
self.serverHost = m.groups()[1]
elif re.match(r'neos_server[\s=]+(\S+):(\d+)',line):
m = re.match(r'neos_server[\s=]+(\S+):(\d+)',line)
self.serverHost = m.groups()[0]
self.serverPort = m.groups()[1]
else:
m = re.match(r'neos_server[\s=]+(\S+)',line)
if m:
self.serverHost = m.groups()[0]
m = re.match(r'kestrel_(job|jobnumber|jobNumber)[\s=]+(\d+)', line)
if m:
self.jobNumber=int(m.groups()[1])
m = re.match(r'kestrel_(pass|password)[\s=]+(\S+)', line)
if m:
self.password = m.groups()[1]
m = re.match(r'socket_timeout[\s=]+(\d+)',line)
if m:
self.socket_timeout = m.groups()[0]
socket.setdefaulttimeout(float(self.socket_timeout))
# options for doCloud
m = re.match(r'docloud_url[\s=]+(\S+)',line)
if m:
self.doCloudUrl = m.groups()[0]
m = re.match(r'docloud_key[\s=]+(\S+)',line)
if m:
self.doCloudKey = m.groups()[0]
m = re.match(r'docloud_prmfile[\s=]+(\S+)',line)
if m:
self.doCloudPrmFile = m.groups()[0]
optfile.close()
self.writeLog("\nFinished reading from \"" + self.optfilename + "\"\n")
else:
raise KestrelSolverException("Could not read options file %s\n" % self.optfilename,self.kestrelGamsSolvers)
def connectServer(self):
#doCloud
if self.doCloudUrl != None:
if import_error:
self.Error(import_error)
if sys.platform == "win32":
os.environ['REQUESTS_CA_BUNDLE'] = os.path.join(os.path.dirname(sys.executable), "GMSPython", "Lib", "site-packages", "certifi", "cacert.pem")
self.doCloudClient = JobClient(self.doCloudUrl, self.doCloudKey)
#neos
else:
if self.logopt in [1,3,4]:
sys.stdout.write("Connecting to: %s://%s:%s\n" % (self.serverProtocol,self.serverHost,self.serverPort))
if self.logopt in [2,4]:
# Append the message to the logfile indicated
try:
f = open(kestrel.logfilename,'a')
f.write("Connecting to: %s://%s:%s\n" % (self.serverProtocol,self.serverHost,self.serverPort))
f.close()
except IOError as e:
self.Fatal("Could not append to log file %s" % self.logfilename)
self.neos = xmlrpc.client.Server("%s://%s:%s" % (self.serverProtocol,self.serverHost,self.serverPort))
reply = self.neos.ping()
if reply.find('alive') < 0:
raise KestrelException("Unable to contact NEOS at https://%s:%d" % \
(self.host, self.port))
def obtainSolvers(self):
#doCloud -> skip
if self.doCloudUrl != None:
return
# Form a list of all kestrel-gams solver available on NEOS
allKestrelSolvers = self.neos.listSolversInCategory("kestrel")
self.kestrelGamsSolvers = []
for s in allKestrelSolvers:
i = s.find(':GAMS')
if i > 0:
self.kestrelGamsSolvers.append(s[0:i])
def checkOptionsFile(self):
if self.solverName and (self.solverName.lower() not in [s.lower() for s in self.kestrelGamsSolvers]):
errmsg = "Solver '%s' not available on NEOS.\n" % self.solverName
raise KestrelSolverException(errmsg, self.kestrelGamsSolvers)
def formSubmission(self):
if not self.solverName:
raise KestrelSolverException("No 'kestrel_solver' option found in option file\n",self.kestrelGamsSolvers)
# Get the matrix, dictionary and instruction file
gamsFiles = {}
gamsFiles['cntr'] = io.BytesIO(self.cntr.encode())
if os.access(self.matrfilename,os.R_OK):
gamsFiles['matr'] = io.BytesIO()
f = open(self.matrfilename,"rb")
zipper = gzip.GzipFile(mode='wb',fileobj=gamsFiles['matr'])
zipper.write(f.read())
zipper.close()
f.close()
if os.access(self.instfilename,os.R_OK):
gamsFiles['inst'] = io.BytesIO()
f = open(self.instfilename,"rb")
zipper = gzip.GzipFile(mode='wb',fileobj=gamsFiles['inst'])
zipper.write(f.read())
zipper.close()
f.close()
if os.access(self.dictfilename,os.R_OK):
gamsFiles['dict'] = io.BytesIO()
f = open(self.dictfilename,"rb")
zipper = gzip.GzipFile(mode='wb',fileobj=gamsFiles['dict'])
zipper.write(f.read())
zipper.close()
f.close()
if self.isMPSGE != 0 and self.modeltype == 5 and os.access(os.path.join(self.scrdir,'gedata.' + self.scrext),os.R_OK): # MCP might be an MPSGE model
gamsFiles['cge'] = io.BytesIO()
f = open(os.path.join(self.scrdir,'gedata.' + self.scrext),"rb")
zipper = gzip.GzipFile(mode='wb',fileobj=gamsFiles['cge'])
s=f.read()
end = s.find(b"gamsdict.")
if end != -1:
start = end
while ord(s[end]) != 32: #whitespace
end = end+1
while ord(s[start]) != 0:
start = start-1
orgStr = s[start+1:end]
replStr = b"./gamsdict.scr" + b" "*(len(orgStr) - len("./gamsdict.scr"))
s = s.replace(orgStr, replStr)
zipper.write(s)
zipper.close()
f.close()
self.xml = """
<document>
<category>kestrel</category>
<solver>%s</solver>
<inputType>GAMS</inputType>
<priority>%s</priority>
""" % (self.solverName,self.priority)
for key in list(gamsFiles.keys()):
self.xml += "<%s><base64>%s</base64></%s>\n" % (key,base64.b64encode(gamsFiles[key].getvalue()).decode(),key)
gamsFiles[key].close()
# Remove 'kestrel', 'neos' and 'socket_timeout' options from options file; they are not needed
email = None
xpressemail = None
runningtime = None
self.xml += "<options><![CDATA["
if self.useOptions:
with open(self.optfilename) as fp:
for line in fp.readlines():
if not re.match(r'kestrel|neos_server|neos_username|neos_user_password|email|xpressemail|runtime|socket_timeout',line):
self.xml += line
elif re.match(r'email',line):
email = line.rsplit()[1]
elif re.match(r'xpressemail',line):
xpressemail = line.rsplit()[1]
elif re.match(r'runtime',line):
runningtime = line.rsplit()[1]
self.xml += "]]></options>\n"
if not email:
email = self.getDefaultEmail()
if not email:
self.Error("No email address provided. Either specify it in an option file or set environment variable NEOS_EMAIL (e.g. via gamsconfig.yaml).")
self.xml += "<email>"
self.xml += email
self.xml += "</email>\n"
if xpressemail:
self.xml += "<xpressemail>"
self.xml += xpressemail
self.xml += "</xpressemail>\n"
if runningtime:
self.xml += "<priority>"
self.xml += runningtime
self.xml += "</priority>"
self.xml += "</document>"
def submit(self):
user = "******" % (os.getenv('LOGNAME'),
socket.getfqdn(socket.gethostname()))
if self.authUsername is None or self.authUserPassword is None:
if self.authUsername: self.writeLog("\nWarning: 'neos_username' was specified, but not 'neos_user_password'")
if self.authUserPassword: self.writeLog("\nWarning: 'neos_user_password' was specified, but not 'neos_username'")
(self.jobNumber,self.password) = \
self.neos.submitJob(self.xml,user,"kestrel")
else:
(self.jobNumber,self.password) = \
self.neos.authenticatedSubmitJob(self.xml,self.authUsername,self.authUserPassword,"kestrel")
if self.jobNumber==0:
raise KestrelException(self.password)
if self.logopt in [1,3,4]:
# Send the output to the screen
sys.stdout.write("\nNEOS job#=%d, pass=%s\n\n" % (self.jobNumber,self.password))
sys.stdout.write("Check the following URL for progress report :\n")
#sys.stdout.write("http://www-neos.mcs.anl.gov/cgi-bin/nph-neos-solver.cgi?admin=results&jobnumber=%d&pass=%s\n\n" % (self.jobNumber,self.password))
sys.stdout.write("%s://%s/neos/cgi-bin/nph-neos-solver.cgi?admin=results&jobnumber=%d&pass=%s\n\n" % (self.serverProtocol,self.serverHost,self.jobNumber,self.password))
if self.logopt in [2,4]:
# Append the error message to the logfile indicated
try:
f = open(self.logfilename,'a')
f.write("\nNEOS job#=%d, pass=%s\n\n" % (self.jobNumber,self.password))
f.write("Check the following URL for progress report :\n")
f.write("%s://%s/neos/cgi-bin/nph-neos-solver.cgi?admin=results&jobnumber=%d&pass=%s\n\n" % (self.serverProtocol,self.serverHost,self.jobNumber,self.password))
f.close()
except IOError as e:
self.Error("Could not append to log file %s" % self.logfilename)
try:
f = open(self.statfilename,'a')
f.write("=1\n\n")
f.write("\nNEOS job#=%d, pass=%s\n\n" % (self.jobNumber,self.password))
f.write("Check the following URL for progress report :\n")
f.write("%s://%s/neos/cgi-bin/nph-neos-solver.cgi?admin=results&jobnumber=%d&pass=%s\n\n" % (self.serverProtocol,self.serverHost,self.jobNumber,self.password))
f.write("=2\n")
f.close()
except IOError as e:
self.Error("Could not append to status file %s\n" % self.statfilename)
def generateMPS(self):
#patch cntr file in order to use option file
try:
f = open(self.cntrfile,'r')
lines = f.readlines()
f.close()
except IOError as e:
self.Fatal("Could not open control file %s" % self.cntrfile)
lines[12] = "101010 1\n"
lines[19] = os.path.join(self.scrdir, "convert.opt\n")
fname = os.path.join(self.scrdir, "gamscntr2.dat")
f = open(fname, "w")
f.writelines(["%s" % i for i in lines])
f.close()
fname = os.path.join(self.scrdir, "convert.opt")
f = open(fname, "w")
f.write("CplexMPS")
f.close()
#win
if sys.platform == "win32":
si = subprocess.STARTUPINFO()
try:
si.dwFlags |= subprocess.STARTF_USESHOWWINDOW
si.wShowWindow = subprocess.SW_HIDE
except:
si.dwFlags |= subprocess._subprocess.STARTF_USESHOWWINDOW
si.wShowWindow = subprocess._subprocess.SW_HIDE
cmdLine = "gmsgennx.exe " + os.path.join(self.scrdir, "gamscntr2.dat") + " convert"
self._p = subprocess.Popen(cmdLine, startupinfo=si, cwd=self.scrdir, creationflags=subprocess.CREATE_NEW_CONSOLE)
exitcode = self._p.wait()
#unix/linux
else:
self._p = subprocess.Popen(["gmsgennx.exe", os.path.join(self.scrdir, "gamscntr2.dat"), "convert"], cwd=self.scrdir)
self.gev = new_gevHandle_tp()
ret = gevCreate(self.gev, GMS_SSSIZE)
if not ret[0]:
raise Exception(ret[1])
if gevInitEnvironmentLegacy(self.gev, os.path.join(self.scrdir, "gamscntr.dat")) != 0:
raise Exception("Error calling gevInitEnvironmentLegacy")
self.gmo = new_gmoHandle_tp()
ret = gmoCreate(self.gmo, GMS_SSSIZE)
if not ret[0]:
raise Exception(ret[1])
gmoRegisterEnvironment(self.gmo, gevHandleToPtr(self.gev))
ret = gmoLoadDataLegacy(self.gmo)
if ret[0] != 0:
raise gams.workspace.GamsException(ret[1])
if gmoModelType(self.gmo) not in [gmoProc_lp, gmoProc_mip, gmoProc_rmip, gmoProc_qcp, gmoProc_miqcp, gmoProc_rmiqcp]:
gmoSolveStatSet(self.gmo, gmoSolveStat_Capability)
gmoModelStatSet(self.gmo, gmoModelStat_NoSolutionReturned)
gmoCompleteSolution(self.gmo)
if gmoUnloadSolutionLegacy(self.gmo) != 0:
gevLogStat(self.gev, '*** Could not write solution')
self.Error("Wrong model type. Supported model types: LP, MIP RMIP, QCP, MIQCP, RMIQCP\n")
input = [ os.path.join(self.scrdir, "cplex.mps"), os.path.join(self.scrdir, "cplex.prm") ]
#create cplex param file
f = open(os.path.join(self.scrdir, "cplex.prm"), "w")
f.write("CPLEX Parameter File Version 12.6.3.0\n")
reslim = str(gevGetIntOpt(self.gev, gevIterLim))
if reslim != GMS_SV_NA:
f.write("CPX_PARAM_ITLIM " + str(reslim) + "\n")
optca = gevGetDblOpt(self.gev, gevOptCA)
if optca != GMS_SV_NA:
f.write("CPX_PARAM_EPAGAP " + str(optca) + "\n")
optcr = gevGetDblOpt(self.gev, gevOptCR)
if optcr != GMS_SV_NA:
f.write("CPX_PARAM_EPGAP " + str(optcr) + "\n")
cutoff = gevGetDblOpt(self.gev, gevCutOff)
if cutoff != GMS_SV_NA and gevGetIntOpt(self.gev, gevUseCutOff):
f.write("CPX_PARAM_CUTLO " + str(cutoff) + "\n")
f.write("CPX_PARAM_CUTUP " + str(cutoff) + "\n")
nodlim = gevGetIntOpt(self.gev, gevNodeLim)
if nodlim != GMS_SV_NA:
f.write("CPX_PARAM_NODELIM " + str(nodlim) + "\n")
cheat = gevGetDblOpt(self.gev, gevCheat)
if cheat != GMS_SV_NA and gevGetIntOpt(self.gev, gevUseCheat):
f.write("CPX_PARAM_OBJDIF " + str(cheat) + "\n")
threads = gevGetIntOpt(self.gev, gevThreadsRaw)
if threads != GMS_SV_NA:
if threads >=0:
f.write("CPX_PARAM_THREADS " + str(threads) + "\n")
else:
threads = max(multiprocessing.cpu_count() + threads, 1)
f.write("CPX_PARAM_THREADS " + str(threads) + "\n")
f.write("CPX_PARAM_MIPDISPLAY 4\n")
if self.doCloudPrmFile:
if os.path.isabs(self.doCloudPrmFile):
prmFile = self.doCloudPrmFile
else:
prmFile = os.path.join(os.path.dirname(self.optfilename), self.doCloudPrmFile)
f2 = open(prmFile, "r")
lines = f2.readlines()[1:]
for l in lines:
f.write(l)
f2.close()
f.close()
logFile = os.path.join(self.scrdir, "doLog.dat")
self.resp = self.doCloudClient.execute(input = input, output = os.path.join(self.scrdir, "results.json"), load_solution=True, log = logFile)
f = open(logFile, "r")
lines = f.readlines()
for l in lines:
gevLogPChar(self.gev, l)
def getResultsDoCloud(self):
sol = json.loads(self.resp.solution.decode("utf-8"))
if sol["CPLEXSolution"]["header"]["solutionTypeString"] == "basic":
gotBasis = True
else:
gotBasis = False
if sol["CPLEXSolution"]["header"]["solutionTypeString"] == "primal":
isPrimal = True
else:
isPrimal = False
mtol = ltol = 0
etypemap = ['E','G','L','X','X','X']
# Do the header info
gmoSetHeadnTail(self.gmo, gmoHobjval, float(sol["CPLEXSolution"]["header"]["objectiveValue"]))
gmoSolveStatSet(self.gmo, 1)
gmoModelStatSet(self.gmo, 1)
if isPrimal:
gmoSetHeadnTail(self.gmo, gmoHmarginals, 0)
# Do the rows
for equ in sol["CPLEXSolution"]["linearConstraints"]:
idx = int(equ["index"])
rhsvalue = gmoGetRhsOne(self.gmo, idx)
level = rhsvalue-float(equ["slack"])
if isPrimal:
dual = GMS_SV_NA
else:
dual = float(equ["dual"])
rowsign = etypemap[gmoGetEquTypeOne(self.gmo,idx)]
if not gotBasis:
if abs(dual) < mtol:
rowindic = gmoBstat_Basic
dual = 0.0
elif ((gmoSense(self.gmo) == gmoObj_Min and dual > 0) or (gmoSense(self.gmo) == gmoObj_Max and dual < 0)) and (abs(float(equ["slack"])) < ltol):
rowindic = gmoBstat_Lower
level = rhsvalue
elif ((gmoSense(self.gmo) == gmoObj_Min and dual < 0) or (gmoSense(self.gmo) == gmoObj_Max and dual > 0)) and (abs(float(equ["slack"])) < ltol):
rowindic = gmoBstat_Upper
level = rhsvalue
else:
rowindic = gmoBstat_Super
else:
if "LL" == equ["status"]:
if rowsign == 'G':
rowindic = gmoBstat_Lower
else:
rowindic = gmoBstat_Upper
elif "BS" == equ["status"]:
rowindic = gmoBstat_Basic
else:
print("Illegal value in rowstat")
rowindic = gmoBstat_Super
gmoSetSolutionEquRec(self.gmo, idx, level, dual, rowindic, gmoCstat_OK)
# Do the columns
for var in sol["CPLEXSolution"]["variables"]:
idx = int(var["index"])
primal = float(var["value"])
if isPrimal:
dual = GMS_SV_NA
else:
dual = float(var["reducedCost"])
xctype = gmoGetVarTypeOne(self.gmo, idx)
if not gotBasis:
lb = gmoGetVarLowerOne(self.gmo, idx)
ub = gmoGetVarUpperOne(self.gmo, idx)
if abs(dual) < mtol:
colindic = gmoBstat_Basic
dual = 0.0
elif ((gmoSense(self.gmo) == gmoObj_Min and dual > 0) or (gmoSense(self.gmo) == gmoObj_Max and dual < 0)) and (abs(primal - lb) < ltol):
colindic = gmoBstat_Lower
primal = lb
elif ((gmoSense(self.gmo) == gmoObj_Min and dual < 0) or (gmoSense(self.gmo) == gmoObj_Max and dual < 0)) and (abs(ub - primal) < ltol):
colindic = gmoBstat_Upper;
x = lb
else:
colindic = gmoBstat_Super
elif var["status"] == "LL":
colindic = gmoBstat_Lower;
elif var["status"] == "BS":
colindic = gmoBstat_Basic;
elif var["status"] == "UL":
colindic = gmoBstat_Upper;
elif var["status"] == "SB":
colindic = gmoBstat_Super;
else:
print("Illegal value in colstat.")
colindic = gmoBstat_Super
if gmoModelType(self.gmo) == gmoProc_mip and xctype != gmovar_X: # For integer or SOS variables always report super basic.
colindic = gmoBstat_Super
gmoSetSolutionVarRec(self.gmo, idx, primal, dual, colindic, gmoCstat_OK)
gmoSetHeadnTail(self.gmo, gmoHresused, time.time() - self.time1)
gmoCompleteSolution(self.gmo)
if gmoUnloadSolutionLegacy(self.gmo) != 0:
gevLogStat(self.gev, '*** Could not write solution')
def getText(self,node):
"""
Returns the text from the node of an xml document
"""
s = ""
if isinstance(node,str):
return node
if isinstance(node.nodeValue,str):
return node.data
elif node.hasChildNodes():
for n in node.childNodes:
s += self.getText(n)
return s
def parseSolution(self,xmlstring):
doc = xml.dom.minidom.parseString(xmlstring)
node = doc.getElementsByTagName('solu')
if node and len(node):
try:
f = open(self.solufilename,'w')
f.write(self.getText(node[0]))
f.close()
except IOError as e:
self.Error("Could not write solution file %s\n" % self.solufilename)
node = doc.getElementsByTagName('stat')
if node and len(node):
try:
f = open(self.statfilename,'w')
f.write(self.getText(node[0]))
f.close()
except IOError as e:
self.Error("Could not write status file %s\n" % self.statfilename)
node = doc.getElementsByTagName('log')
if node and len(node):
if self.logopt in [1,3,4]:
# Send the output to the screen
sys.stdout.write(self.getText(node[0]))
if self.logopt in [2,4]:
# Append the error message to the logfile indicated
try:
f = open(self.logfilename,'a')
f.write(self.getText(node[0]))
f.close()
except IOError as e:
self.Error("Could not append log file %s\n" % self.logfilename)
doc.unlink()
def getResults(self):
offset = 0
status = self.neos.getJobStatus(self.jobNumber,self.password)
try:
while (status == "Waiting" or status=="Running"):
(results,offset) = self.neos.getIntermediateResults(self.jobNumber, self.password,offset)
if isinstance(results,xmlrpc.client.Binary):
results = results.data.decode()
if results and len(results):
if self.logopt in [1,3,4]:
# Send the output to the screen
sys.stdout.write(results)
if self.logopt in [2,4]:
# Append the error message to the logfile indicated
try:
f = open(self.logfilename,'a')
f.write(results)
f.close()
except IOError as e:
self.Error("Could not append to log file %s" % self.logfilename)
try:
f = open(self.statfilename,'a')
f.write("=1\n\n")
f.write(results)
f.write("=2\n")
f.close()
except IOError as e:
self.Error("Could not append to status file %s\n" % self.statfilename)
status = self.neos.getJobStatus(self.jobNumber,self.password)
time.sleep(5)
except KeyboardInterrupt as e:
msg = '''Keyboard Interrupt\n\
Job is still running on remote machine\n\
To retrieve results, run GAMS using solver 'kestrel' with option file:\n\
kestrel_job %d\n\
kestrel_pass %s\n\n\
To stop job, run GAMS using solver 'kestrelkil' with above option file\n\
''' % (self.jobNumber, self.password)
self.Error(msg)
resultsXML = self.neos.getFinalResults(self.jobNumber,self.password)
if isinstance(resultsXML,xmlrpc.client.Binary):
resultsXML = resultsXML.data
self.parseSolution(resultsXML)
def solving_placement_problem_from_file(topology_graph, request_graph,
test_num, CPLEX_PATH,
cplex_models_path, results_path,
locally):
# Reading networkx file
G_topology = read_json_file(topology_graph)
G_request = read_json_file(request_graph)
set_PM = list(G_topology.nodes)
set_state_or_nf = list(G_request.nodes)
set_state, set_nf, set_replica = [], [], []
for i in set_state_or_nf:
if "function" in i:
set_nf.append(i)
elif "state" in i:
set_state.append(i)
elif "replica" in i:
set_replica.append(i)
if not os.path.isfile("{}/p5_cplex_model_{}_2.lp".format(
cplex_models_path, test_num)):
cplex_f = open('{}/p5_cplex_model_{}_2.lp'.format(
cplex_models_path, test_num),
mode='a')
# TODO: Validating request graph
for i in set_state:
try:
G_request.nodes[i]['size']
except:
RuntimeError(
"The given request graph is incorrect: State {} has no 'size' value"
.format(i))
s = {i: G_request.nodes[i]['size'] for i in set_state + set_replica}
c = {i: G_topology.nodes[i]['capacity'] for i in set_PM}
print("Generating delay matrix...")
d = generating_delay_matrix(G_topology)
print("Generating state-function adjacency matrix...")
e_r = generating_req_adj(set_state, set_nf + set_replica, G_request)
print("Generating Function mapping matrix...")
M = generating_nf_mapping_matrix(G_topology)
print("Generating Anti-Affinity set")
AA = generating_AA(set_state, G_request)
print("Generating OR-Link set")
OL = generating_OL(set_state, set_nf, set_replica, G_request)
# ## Into File ############################################################################################
cplex_f = open('{}/p5_cplex_model_{}_2.lp'.format(
cplex_models_path, test_num),
mode='a')
cplex_f.write("Minimize\n obj: [ ")
servers = [i for i in set_PM if "server" in i]
server_permutations = list(itertools.permutations(servers, 2))
first = True
for i, j in server_permutations:
asd = list(
itertools.permutations(set_state + set_replica + set_nf, 2))
for u, v in asd:
if (e_r[u, v] * d[i, j] * 2 > 0):
if first:
cplex_f.write(
" {} y_({},{})_({},{})*z_({},{})\n".format(
e_r[u, v] * d[i, j] * 2, i, u, j, v, u, v))
first = False
else:
cplex_f.write(
" + {} y_({},{})_({},{})*z_({},{})\n".format(
e_r[u, v] * d[i, j] * 2, i, u, j, v, u, v))
cplex_f.close()
cplex_f = open('{}/p5_cplex_model_{}_2.lp'.format(
cplex_models_path, test_num),
mode='rb+')
cplex_f.seek(-1, os.SEEK_END)
cplex_f.truncate()
cplex_f = open('{}/p5_cplex_model_{}_2.lp'.format(
cplex_models_path, test_num),
mode='a')
cplex_f.write("]/2 \n")
# contraint 1 --------------------------------------------------------------------------------------------
print("Generating mapping constraints")
cplex_f.write("\nSubject To \n")
for u in set_state + set_replica:
c_name = "c1_{}".format(u)
cplex_f.write(" {}: ".format(c_name))
for i in set_PM:
cplex_f.write(" x_({},{}) +".format(i, u))
cplex_f.close()
cplex_f = open('{}/p5_cplex_model_{}_2.lp'.format(
cplex_models_path, test_num),
mode='rb+')
cplex_f.seek(-2, os.SEEK_END)
cplex_f.truncate()
cplex_f = open('{}/p5_cplex_model_{}_2.lp'.format(
cplex_models_path, test_num),
mode='a')
cplex_f.write(" = 1\n")
# contraint 2 --------------------------------------------------------------------------------------------
print("Generating capacity constraints")
for i in set_PM:
c_name = "c2_{}".format(i)
cplex_f.write(" {}: ".format(c_name))
for u in set_state + set_replica:
cplex_f.write("{} x_({},{}) +".format(s[u], i, u))
cplex_f.close()
cplex_f = open('{}/p5_cplex_model_{}_2.lp'.format(
cplex_models_path, test_num),
mode='rb+')
cplex_f.seek(-2, os.SEEK_END)
cplex_f.truncate()
cplex_f = open('{}/p5_cplex_model_{}_2.lp'.format(
cplex_models_path, test_num),
mode='a')
cplex_f.write(" <= {}\n".format(c[i]))
# contraint 3 --------------------------------------------------------------------------------------------
print("Generating AA constraints")
for i in set_PM:
if "server" in i:
for u, v in AA:
c_name = "c3_{}_{}_in_{}".format(u, v, i)
cplex_f.write(" {}: ".format(c_name))
cplex_f.write(" x_({},{}) + x_({}, {}) <= 1\n".format(
i, u, i, v))
# contraint 4 --------------------------------------------------------------------------------------------
print("Generating NF mapping constraints")
for function in set_nf:
for server in set_PM:
c_name = "c4_{}_in_{}".format(function, server)
try:
if M[function] == server:
cplex_f.write(" {}: ".format(c_name))
cplex_f.write(" x_({},{}) = 1\n".format(
server, function))
else:
cplex_f.write(" {}: ".format(c_name))
cplex_f.write(" x_({},{}) = 0\n".format(
server, function))
except:
cplex_f.write(" {}: ".format(c_name))
cplex_f.write(" x_({},{}) = 0\n".format(server, function))
# contraint 5 --------------------------------------------------------------------------------------------
def for_multiprocessing(list, from_, to_, test_num, process_id,
cplex_models_path):
print("Starting process {}, from: {}, to:{}".format(
process_id, from_, to_))
c5_f = open('{}/c5_testnum{}_tmp{}_2.txt'.format(
cplex_models_path, test_num, process_id),
mode='a')
start = from_
for i, j in list[from_:to_]:
if (start % 10000) == 0:
print("{}: {}".format(process_id, start))
c_name = "c5_({},{})_({},{})_0".format(i[0], i[1], j[0], j[1])
c5_f.write(" {}: ".format(c_name))
c5_f.write(" y_({},{})_({},{}) >= 0 \n".format(
i[0], i[1], j[0], j[1]))
c_name = "c5_({},{})_({},{})_1".format(i[0], i[1], j[0], j[1])
c5_f.write(" {}: ".format(c_name))
c5_f.write(
" y_({},{})_({},{}) - x_({},{}) - x_({},{}) >= -1 \n".
format(i[0], i[1], j[0], j[1], i[0], i[1], j[0], j[1]))
start += 1
c5_f.close()
print("Ending process {}".format(process_id))
print(datetime.datetime.now())
print("Generating QP -> ILP transformation constraints")
index_set = set()
for i in set_PM:
for u in set_state + set_replica + set_nf:
index_set.add((i, u))
index_combinations = list(itertools.permutations(index_set, 2))
index_combinations_size = len(index_combinations)
print("Size of this contraints: {}".format(index_combinations_size))
from_to_list = []
core_num = 10
core_job_count = len(index_combinations) / 10
for i in range(core_num):
from_ = i * core_job_count
to_ = (i + 1) * core_job_count
if i == core_num - 1:
to_ = index_combinations_size
from_to_list.append((from_, to_))
import multiprocessing
processes = []
for i in range(0, core_num):
p = multiprocessing.Process(
target=for_multiprocessing,
args=(index_combinations, from_to_list[i][0],
from_to_list[i][1], test_num, i, cplex_models_path))
processes.append(p)
p.start()
for process in processes:
process.join()
tempfiles = [
"{}/c5_testnum{}_tmp{}_2.txt".format(cplex_models_path, test_num,
i) for i in range(core_num)
]
for tempfile in tempfiles:
#print("Adding file {}".format(tempfile))
tmp_f = open(tempfile, "r")
cplex_f.write(tmp_f.read())
for tempfile in tempfiles:
os.remove(tempfile)
# contraint 6 --------------------------------------------------------------------------------------------
print("Generating 'Does it matter?' constraints")
for u in (set_state + set_nf + set_replica):
for v in (set_state + set_nf + set_replica):
if u != v:
if is_OL(u, v, OL):
c_name = "c6_({},{})_0".format(u, v)
cplex_f.write("\n {}: ".format(c_name))
first = True
for i, j in get_OLs(u, v, OL):
if first:
cplex_f.write(" z_({},{})".format(i, j))
first = False
else:
cplex_f.write(" + z_({},{})".format(i, j))
cplex_f.write(" = 1 \n")
else:
c_name = "c6_({},{})_1".format(u, v)
if "function" in u and "replica" in v:
cplex_f.write("\n {}: ".format(c_name))
cplex_f.write(" z_({},{}) = 0".format(u, v))
elif "replica" in u and "state" in v:
cplex_f.write("\n {}: ".format(c_name))
cplex_f.write(" z_({},{}) = 0".format(u, v))
else:
cplex_f.write("\n {}: ".format(c_name))
cplex_f.write(" z_({},{}) = 1".format(u, v))
# Bounds --------------------------------------------------------------------------------------------
cplex_f.write("\nBounds\n")
for i in set_PM:
for u in set_state + set_replica + set_nf:
cplex_f.write("0 <= x_({},{}) <= 1\n".format(i, u))
index_set = set()
for i in set_PM:
for u in set_state + set_replica + set_nf:
index_set.add((i, u))
index_permutations = list(itertools.permutations(index_set, 2))
for i, j in index_permutations:
cplex_f.write("0 <= y_({},{})_({},{}) <= 1\n".format(
i[0], i[1], j[0], j[1]))
index_permutations = list(
itertools.permutations((set_state + set_nf + set_replica), 2))
for u, v in index_permutations:
cplex_f.write("0 <= z_({},{}) <= 1\n".format(u, v))
# Binaries --------------------------------------------------------------------------------------------
cplex_f.write("\nBinaries\n")
for i in set_PM:
for u in set_state + set_replica + set_nf:
cplex_f.write(" x_({},{})\n".format(i, u))
index_set = set()
for i in set_PM:
for u in set_state + set_replica + set_nf:
index_set.add((i, u))
index_permutations = list(itertools.permutations(index_set, 2))
for i, j in index_permutations:
cplex_f.write(" y_({},{})_({},{})\n".format(
i[0], i[1], j[0], j[1]))
index_permutations = list(
itertools.permutations((set_state + set_nf + set_replica), 2))
for u, v in index_permutations:
cplex_f.write(" z_({},{})\n".format(u, v))
time.sleep(2)
cplex_f.write("End\n")
cplex_f.close()
########################################################################################################
if not os.path.isfile(CPLEX_PATH):
raise RuntimeError('CPLEX does not exist ({})'.format(CPLEX_PATH))
if not os.path.isfile("{}/p5_cplex_model_{}_2.sav".format(
cplex_models_path, test_num)):
subprocess.call(
"{} -c 'read {}/p5_cplex_model_{}_2.lp' 'write {}/p5_cplex_model_{}_2.sav sav'"
.format(CPLEX_PATH, cplex_models_path, test_num, cplex_models_path,
test_num),
shell=True)
t1 = datetime.datetime.now()
cost = 0
mapping_result = {i: "" for i in set_state + set_nf + set_replica}
if locally:
# solving problem in locally
print("\n\nSolving the problem locally - 2")
subprocess.call(
"{} -c 'read {}/p5_cplex_model_{}_2.mps' 'optimize' 'write {}/p5_cplex_result_{}_2 sol'"
.format(CPLEX_PATH, cplex_models_path, test_num, results_path,
test_num),
shell=True)
else:
print("\n\nSolving the problem remotely in the IBM cloud - 2")
if not os.path.isfile("{}/p5_cplex_result_{}_2".format(
results_path, test_num)):
client = JobClient(
"https://api-oaas.docloud.ibmcloud.com/job_manager/rest/v1/",
"api_e7f3ec88-92fd-4432-84d7-f708c4a33132")
print(
"You can check the status of the problem procesing here: https://dropsolve-oaas.docloud.ibmcloud.com/dropsolve"
)
resp = client.execute(input=[
"{}/p5_cplex_model_{}_2.sav".format(cplex_models_path,
test_num)
],
output="{}/p5_cplex_result_{}_2".format(
results_path, test_num))
if resp.job_info["solveStatus"] == "INFEASIBLE_SOLUTION":
print("There is no valid mapping!")
return 0
def is_json_file(report_input_file):
with open(report_input_file) as unknown_file:
c = unknown_file.read(1)
if c != '<':
return True
return False
t2 = datetime.datetime.now()
if is_json_file("{}/p5_cplex_result_{}_2".format(results_path, test_num)):
with open("{}/p5_cplex_result_{}_2".format(results_path,
test_num)) as f:
result = json.load(f)
for i in result["CPLEXSolution"]["variables"]:
if ("x_" in list(i["name"])) and i["value"] == str(1):
# print("{} = 1".format(i["name"]))
server = i["name"].split(',')[0][3:]
ve = i["name"].split(',')[1][:-1]
mapping_result[ve] = server
print("*** Delay cost: {} ***".format(
result["CPLEXSolution"]["header"]["objectiveValue"]))
cost = result["CPLEXSolution"]["header"]["objectiveValue"]
else:
with open("{}/p5_cplex_result_{}_2".format(results_path, test_num),
'r') as file:
xml_result = file.read().replace('\n', '')
result = xmltodict.parse(xml_result)
print("*** Delay cost: {} ***".format(
result["CPLEXSolution"]["header"]["@objectiveValue"]))
cost = result["CPLEXSolution"]["header"]["@objectiveValue"]
running_time = t2 - t1
print("RUNNING TIME: {}".format(running_time))
return cost, mapping_result, running_time
from docloud.job import JobClient
if __name__ == '__main__':
url = "Paste your base URL"
api_key = "Paste your api key"
client = JobClient(url, api_key)
resp = client.execute(input=["models/truck.dat", "models/truck.mod"],
output="results.json")
* cplex_config.py
* cplex_config_<hostname>.py
* docloud_config.py (must only contain context.solver.docloud configuration)
These files contain the credentials and other properties. For example,
something similar to::
context.solver.docloud.url = 'https://docloud.service.com/job_manager/rest/v1'
context.solver.docloud.key = 'example api_key'
'''
url = 'https://api-oaas.docloud.ibmcloud.com/job_manager/rest/v1'
key = 'example api_key'
# if url is None or key is None:
# create a default context and use credentials defined in there.
# context = Context.make_default_context()
# url = context.solver.docloud.url
# key = context.solver.docloud.key
client = JobClient(url=url, api_key=key)
resp = client.execute(input=[
'NASL_MP.py', 'data/teams.csv', 'data/constraint_detail.csv',
'data/dates.csv', 'data/distances.json'
],
output='solution.json',
waittime=60,
load_solution=True,
log='logs.txt')
# In[ ]:
from docloud.job import JobClient
if __name__ == '__main__':
url = "Paste your base URL"
api_key = "Paste your api key"
client = JobClient(url, api_key)
with open("models/truck.mod", "rb") as modFile:
resp = client.execute(input=[{"name":"truck.mod",
"file":modFile},
"models/truck.dat"],
output="results.json",
log="solver.log",
gzip=True,
waittime=300,
delete_on_completion=True)
def solving_placement_problem_from_file(topology_graph, request_graph,
test_num):
if not os.path.isfile(
"./cplex_models/p5_cplex_model_{}.lp".format(test_num)):
cplex_f = open('./cplex_models/p5_cplex_model_{}.lp'.format(test_num),
mode='a')
# Reading networkx file
G_topology = read_json_file(topology_graph)
G_request = read_json_file(request_graph)
set_PM = list(G_topology.nodes)
set_state_or_nf = list(G_request.nodes)
set_state, set_nf, set_replica = [], [], []
for i in set_state_or_nf:
if "function" in i:
set_nf.append(i)
elif "state" in i:
set_state.append(i)
elif "replica" in i:
set_replica.append(i)
# TODO: Validating request graph
for i in set_state:
try:
G_request.nodes[i]['size']
except:
RuntimeError(
"The given request graph is incorrect: State {} has no 'size' value"
.format(i))
s = {i: G_request.nodes[i]['size'] for i in set_state + set_replica}
c = {i: G_topology.nodes[i]['capacity'] for i in set_PM}
print("Generating delay matrix...")
d = generating_delay_matrix(G_topology)
print("Generating state-function adjacency matrix...")
e_r = generating_req_adj(set_state, set_nf + set_replica, G_request)
print("Generating Function mapping matrix...")
M = generating_nf_mapping_matrix(G_topology)
print("Generating Anti-Affinity set")
AA = generating_AA(set_state, G_request)
print("Generating OR-Link set")
OL = generating_OL(set_state, set_nf, set_replica, G_request)
opt_model = cpx.Model(name="P5")
# Binary variables
print("Creating variables 1...")
x_vars = {(i, u): opt_model.binary_var(name="x_({0},{1})".format(i, u))
for i in set_PM for u in set_state + set_replica + set_nf}
print("Creating variables 2...")
index_set = set()
for i in set_PM:
for u in set_state + set_replica + set_nf:
index_set.add((i, u))
index_permutations = list(itertools.permutations(index_set, 2))
y_vars = {(i[0], i[1], j[0], j[1]): opt_model.binary_var(
name="y_({},{})_({},{})".format(i[0], i[1], j[0], j[1]))
for i, j in index_permutations}
print("Creating variables 3...")
index_permutations = list(
itertools.permutations((set_state + set_nf + set_replica), 2))
z_vars = {(u, v): opt_model.binary_var(name="z_({},{})".format(u, v))
for u, v in index_permutations}
# == constraints 1 - virtual element can be mapped into only one server
print(
"Creating constraints 1 - virtual element can be mapped into only one server"
)
for u in set_state + set_replica:
c_name = "c1_{}".format(u)
opt_model.add_constraint(ct=opt_model.sum(x_vars[i, u]
for i in set_PM) == 1,
ctname=c_name)
# <= constraints 2 - server capacity constraint
print("Creating constraints 2 - server capacity constraint")
for i in set_PM:
c_name = "c2_{}".format(i)
opt_model.add_constraint(
ct=opt_model.sum(s[u] * x_vars[i, u]
for u in set_state + set_replica) <= c[i],
ctname=c_name)
# <= constraints 3 - anti-affinity rules
print("Creating constraints 3 - anti-affinity rules")
for i in set_PM:
if "server" in i:
for u, v in AA:
c_name = "c3_{}_{}_in_{}".format(u, v, i)
opt_model.add_constraint(
ct=(x_vars[i, u] + x_vars[i, v]) <= 1, ctname=c_name)
# == constraints 4 - NFs running places
print("Creating constraints 4 - NFs running places")
for function in set_nf:
for server in set_PM:
c_name = "c4_{}_in_{}".format(function, server)
try:
if M[function] == server:
opt_model.add_constraint(ct=x_vars[server,
function] == 1,
ctname=c_name)
# print("x_vars[{}, {}] == 1".format(server, function))
else:
opt_model.add_constraint(ct=x_vars[server,
function] == 0,
ctname=c_name)
# print("x_vars[{}, {}] == 0".format(server, function))
except:
opt_model.add_constraint(ct=x_vars[server, function] == 0,
ctname=c_name)
# print("x_vars[{}, {}] == 0".format(server, function))
# >= constraints 5 - QP -> ILP transformation constraints
print("Creating constraints 5 - QP -> ILP transformation constraints")
index_set = set()
for i in set_PM:
for u in set_state + set_replica + set_nf:
index_set.add((i, u))
index_combinations = list(itertools.permutations(index_set, 2))
for i, j in index_combinations:
c_name = "c5_({},{})_({},{})_0".format(i[0], i[1], j[0], j[1])
opt_model.add_constraint(ct=y_vars[i[0], i[1], j[0], j[1]] >= 0,
ctname=c_name)
c_name = "c5_({},{})_({},{})_1".format(i[0], i[1], j[0], j[1])
opt_model.add_constraint(
ct=y_vars[i[0], i[1], j[0], j[1]] >=
(x_vars[i[0], i[1]] + x_vars[j[0], j[1]] - 1),
ctname=c_name)
print("Creating constraints 6 - z variable rules")
for u in (set_state + set_nf + set_replica):
for v in (set_state + set_nf + set_replica):
if u != v:
if is_OL(u, v, OL):
c_name = "c6_({},{})_0".format(u, v)
opt_model.add_constraint(ct=opt_model.sum(
z_vars[(i, j)] for i, j in get_OLs(u, v, OL)) == 1,
ctname=c_name)
else:
c_name = "c6_({},{})_1".format(u, v)
if "function" in u and "replica" in v:
opt_model.add_constraint(ct=z_vars[(u, v)] == 0,
ctname=c_name)
elif "replica" in u and "state" in v:
opt_model.add_constraint(ct=z_vars[(u, v)] == 0,
ctname=c_name)
else:
opt_model.add_constraint(ct=z_vars[(u, v)] == 1,
ctname=c_name)
print("Creating Objective function...")
servers = [i for i in set_PM if "server" in i]
server_permutations = list(itertools.permutations(servers, 2))
objective = opt_model.sum(
y_vars[i, u, j, v] * e_r[u, v] * d[i, j] * z_vars[u, v]
for i, j in server_permutations for u, v in list(
itertools.permutations(set_state + set_replica + set_nf, 2)))
# # # for minimization
opt_model.minimize(objective)
print("Exporting the problem")
opt_model.export_as_lp(basename="p5_cplex_model_{}".format(test_num),
path="./cplex_models")
subprocess.call(
"/home/epmetra/projects/cplex/cplex/bin/x86-64_linux/cplex -c 'read /home/epmetra/projects/LO/cplex_models/p5_cplex_model_{}.lp' 'write /home/epmetra/projects/LO/cplex_models/p5_cplex_model_{}.mps mps'"
.format(test_num, test_num),
shell=True)
# solving with local cplex
# print("Solving the problem locally")
# print(datetime.datetime.now())
# asd = opt_model.solve()
# solving in the docplex cloud
print("Solving the problem by the cloud - 1")
print(datetime.datetime.now())
if not os.path.isfile(
"optimization_results/p5_cplex_result_{}.json".format(test_num)):
client = JobClient(
"https://api-oaas.docloud.ibmcloud.com/job_manager/rest/v1/",
"api_e7f3ec88-92fd-4432-84d7-f708c4a33132")
print(
"You can check the status of the problem procesing here: https://dropsolve-oaas.docloud.ibmcloud.com/dropsolve"
)
resp = client.execute(
input=["./cplex_models/p5_cplex_model_{}.mps".format(test_num)],
output="optimization_results/p5_cplex_result_{}.json".format(
test_num))
mapping_result = {i: "" for i in set_state + set_nf + set_replica}
if resp.job_info["solveStatus"] == "INFEASIBLE_SOLUTION":
print("There is no valid mapping!")
return 0
else:
with open("./optimization_results/p5_cplex_result_{}.json".format(
test_num)) as f:
result = json.load(f)
for i in result["CPLEXSolution"]["variables"]:
if ("x_" in i["name"]) and i["value"] == str(1):
print("{} = 1".format(i["name"]))
server = i["name"].split(',')[0][3:]
ve = i["name"].split(',')[1][:-1]
mapping_result[ve] = server
print("*** Delay cost: {} ***".format(
result["CPLEXSolution"]["header"]["objectiveValue"]))
return result["CPLEXSolution"]["header"][
"objectiveValue"], mapping_result
else:
with open("./optimization_results/p5_cplex_result_{}.json".format(
test_num)) as f:
result = json.load(f)
for i in result["CPLEXSolution"]["variables"]:
if ("x_" in i["name"]) and i["value"] == str(1):
print("{} = 1".format(i["name"]))
print("*** Delay cost: {} ***".format(
result["CPLEXSolution"]["header"]["objectiveValue"]))
return result["CPLEXSolution"]["header"]["objectiveValue"]
print(results)
# >>>> 3. docplexcloud
# key = #DOCPLEX API KEY (get from docloud website)
# base_url = #DOCPLEX URL (get from docloud website)
# Dependencies : docloud (pip install docloud)
import json
import glob
import pandas
from docloud.job import JobClient
client = JobClient(base_url, key)
model.write("temp.lp", io_options={"symbolic_solver_labels": True})
# with open("temp.lp") as lpfile:
# resp = client.execute(input=lpfile,output=None,load_solution = True)
file = glob.glob("temp.lp")
resp = client.execute(input=file, output=None, load_solution=True)
solution = json.loads(resp.solution.decode("utf-8"))
#os.remove(file)
for i, k in solution['CPLEXSolution']['header'].items():
print(i, ':', k)
results = pandas.DataFrame(solution['CPLEXSolution']['variables']).filter(
items=['index', 'name', 'value', 'status'])
print(results)
# >>>> 4. GUROBI
## TBD
exit()
url = "Paste your base URL"
api_key = "Paste your api key"
client = JobClient(url, api_key)
factory = ProblemFactory()
pb = factory.createSampleProblem()
# encode the problem using the specified encoder (which extends json.JSONEncoder)
data = json.dumps(pb, cls=ProblemEncoder).encode('utf-8')
# submit the model truck.mod for execution,
# using the Problem encoded as JSON for data
# then download result once the execution is done
# and finally delete the job
resp = client.execute(
input=["models/truck.mod", {
'name': "truck.json",
'data': data
}],
output="results.json",
gzip=True,
load_solution=True,
delete_on_completion=True)
# decode the solution using the solution decoder
result = json.loads(resp.solution.decode("utf-8"),
object_hook=solution_decoder)
result.displaySolution()
from docloud.job import JobClient
if __name__ == '__main__':
url = "Paste your base URL"
api_key = "Paste your api key"
client = JobClient(url, api_key)
resp = client.execute(input=["models/truck.dat",
"models/truck.mod"],
output="results.json")
from docloud.job import JobClient
if __name__ == '__main__':
url = "Paste your base URL"
api_key = "Paste your api key"
client = JobClient(url, api_key)
with open("models/truck.mod", "rb") as modFile:
resp = client.execute(input=[{
"name": "truck.mod",
"file": modFile
}, "models/truck.dat"],
output="results.json",
log="solver.log",
gzip=True,
waittime=300,
delete_on_completion=True)
the following files:
* cplex_config.py
* cplex_config_<hostname>.py
* docloud_config.py (must only contain context.solver.docloud configuration)
These files contain the credentials and other properties. For example,
something similar to::
context.solver.docloud.url = 'https://docloud.service.com/job_manager/rest/v1'
context.solver.docloud.key = 'example api_key'
'''
url = None
key = None
if url is None or key is None:
# create a default context and use credentials defined in there.
context = Context.make_default_context()
url = context.solver.docloud.url
key = context.solver.docloud.key
client = JobClient(url=url, api_key=key)
resp = client.execute(input=[
'diet_pandas.py', 'diet_food.csv', 'diet_nutrients.csv',
'diet_food_nutrients.csv'
],
output='solution.json',
load_solution=True,
log='logs.txt')
def run(choice, threshold, confidence_interval, facility_number,
min_threshold):
"""
This solves the corresponding optimization problem to cplex cloud.
It takes 5 parameters:
i) choice : An integer , (1-5) which represents the optimization model choice of the user
ii) threshold(m) : An integer , which represents the maximum possible distance between two districts which are counted as appropriate_pairs , meter
iii) confidence_interval : An integer , (0-100) which represents the confidence level of the stochastic model
iv) facility_number : An integer , (0 - NUMBER_OF_DISTRICT) which represents the maximum fire station number for maximum coverage models
v) min_threshold(min) : An integer , which represents the maximum traveling time between two districts in terms of minutes.
It returns nothing but it creates a .txt file that contains the solution of the corresponding optimization problem.
"""
#Create a clinet
client = JobClient(CPLEX_BASE_URL, CPLEX_API_KEY)
# Choice == 1 , base model
if choice == 1:
resp = client.execute(input=[
"Mod_Files/BaseModel.mod",
"Mod_Files/BaseModel_" + str(threshold) + ".dat"
],
output="Solutions/BaseModel_Sol_" +
str(threshold) + ".txt")
return "Solutions/BaseModel_Sol_" + str(threshold) + ".txt"
# Choice == 2 , max_coverage model
if choice == 2:
resp = client.execute(input=[
"Mod_Files/MultiCoverage.mod",
"Mod_Files/MultiCoverage_" + str(threshold) + ".dat"
],
output="Solutions/MultiCoverage_Sol_" +
str(threshold) + ".txt")
return "Solutions/MultiCoverage_Sol_" + str(threshold) + ".txt"
# Choice == 3 , max coverage
if choice == 3:
resp = client.execute(input=[
"Mod_Files/MaxCoverage.mod", "Mod_Files/MaxCoverage_" +
str(threshold) + "_" + str(facility_number) + ".dat"
],
output="Solutions/MaxCoverage_Sol_" +
str(threshold) + "_" + str(facility_number) +
".txt")
return "Solutions/MaxCoverage_Sol_" + str(threshold) + "_" + str(
facility_number) + ".txt"
# Choice == 4 , stochastic_coverage
if choice == 4:
resp = client.execute(input=[
"Mod_Files/StochasticCoverage.mod",
"Mod_Files/Stochastic_Coverage_" + str(min_threshold) + "_" +
str(confidence_interval) + ".dat"
],
output="Solutions/Stochastic_Coverage_Sol_" +
str(min_threshold) + "_" +
str(confidence_interval) + ".txt")
return "Solutions/Stochastic_Coverage_Sol_" + str(
min_threshold) + "_" + str(confidence_interval) + ".txt"
# Choice == 5 , stochastic max coverage
if choice == 5:
resp = client.execute(input=[
"Mod_Files/MaxCoverage.mod",
"Mod_Files/Stochastic_MaxCoverage_" + str(min_threshold) + "_" +
str(facility_number) + "_" + str(confidence_interval) + ".dat"
],
output="Solutions/Stochastic_MaxCoverage_Sol_" +
str(min_threshold) + "_" + str(facility_number) +
"_" + str(confidence_interval) + ".txt")
return "Solutions/Stochastic_MaxCoverage_Sol_" + str(
min_threshold) + "_" + str(facility_number) + "_" + str(
confidence_interval) + ".txt"
class Optimizer(object):
'''
Handles the actual optimization task.
Creates and executes a job builder for an optimization problem instance.
Encapsulates the DOCloud API.
This class is designed to facilitate multiple calls to the optimizer, such as would occur in a decomposition algorithm,
although it transparently supports single use as well.
In particular, the data can be factored into a constant data set that does not vary from run to run (represented by a JSON or .dat file)
and a variable piece that does vary (represented by a Collector object).
The optimization model can also be factored into two pieces, a best practice for large models and multi-models:
A data model that defines the tuples and tuple sets that will contain the input and output data.
An optimization model that defines the decision variables, decision expressions, objective function,
constraints, and pre- and post-processing data transformations.
Factoring either the data or the optimization model in this fashion is optional.
The problem instance is specified by the OPL model and input data received from the invoking (e.g. ColumnGeneration) instance.
Input and output data are realized as instances of OPLCollector, which in turn are specified by their respective schemas.
This class is completely independent of the specific optimization problem to be solved.
'''
def __init__(self, problemName, model=None, resultDataModel=None, credentials=None, *attachments):
'''
Constructs an Optimizer instance.
The instance requires an optimization model as a parameter.
You can also provide one or more data files as attachments, either in OPL .dat or in JSON format. This data does not
change from solve to solve. If you have input data that does change, you can provide it to the solve method as an OPLCollector object.
:param problemName: name of this optimization problem instance
:type problemName: String
:param model: an optimization model written in OPL
:type model: Model.Source object or String
:param resultDataModel: the application data model for the results of the optimization
:type resultDataModel: dict<String, StructType>
:param credentials: DOcplexcloud url and api key
:type credentials: {"url":String, "key":String}
:param attachments: URLs for files representing the data that does not vary from solve to solve
:type attachments: list<URL>
'''
self.name = problemName
self.model = model
self.resultDataModel = resultDataModel
self.attachData(attachments)
self.streamsRegistry = []
self.history = []
self.credentials = credentials
self.jobclient = JobClient(credentials["url"], credentials["key"]);
self.solveStatus = JobSolveStatus.UNKNOWN;
def getName(self):
"""
Returns the name of this problem
"""
return self.name
def setOPLModel(self, name, dotMods=None, modelText=None):
'''
Sets the OPL model.
This method can take any number of dotMod arguments, but
there are two common use cases:
First, the optimization model can be composed of two pieces:
A data model that defines the tuples and tuple sets that will contain the input and output data.
An optimization model that defines the decision variables, decision expressions, objective function,
constraints, and pre- and post-processing data transformations.
The two are concatenated, so they must be presented in that order.
If such a composite model is used, you do not need to import the data model into the optimization model using an OPL include statement.
Second, you do not have to use a separate data model, in which case a single dotMod must be provided
which encompasses both the data model and the optimization model.
@param name: the name assigned to this OPL model (should have the format of a file name with a .mod extension)
@type name: String
@param dotMods: URLs pointing to OPL .mod files, which will be concatenated in the order given
@type dotMods: List<URL>
@param modelText: the text of the OPL model, which will be concatenated in the order given
@type modelText: List<String>
@return this optimizer
@raise ValueError if a model has already been defined or if dotMods or modelText is empty
'''
if self.model is not None:
raise ValueError("model has already been set")
self.model = ModelSource(name=name, dotMods=dotMods, modelText=modelText)
return self
def setResultDataModel(self, resultDataModel):
'''
Sets the application data model for the results of the optimization
@param resultDataModel: the application data model for the results of the optimization
@type resultDataModel: dict<String, StructType>
'''
if self.resultDataModel is not None:
raise ValueError("results data model has already been defined")
self.resultDataModel = resultDataModel
return self
def attachData(self, attachments):
'''
Attaches one or more data files, either in OPL .dat or in JSON format. This data does not
change from solve to solve. If you have input data that does change, you can provide it as a Collector object.
@param attachments: files representing the data that does not vary from solve to solve
@type attachments: list<URL>
@return this optimizer
@raise ValueError if an item of the same name has already been attached
'''
self.attachments = {}
if attachments is not None:
for f in attachments:
fileName = os.path.splitext(os.path.basename(urlparse(f)))[0]
if fileName in self.attachments:
raise ValueError(fileName + " already attached")
self.attachments[fileName] = f
return self;
def solve(self, inputData=None, solutionId=""):
'''
Solves an optimization problem instance by calling the DOCloud solve service (Oaas).
Creates a new job request, incorporating any changes to the variable input data,
for a problem instance to be processed by the solve service.
Once the problem is solved, the results are mapped to an instance of an OPL Collector.
Note: this method will set a new destination for the JSON serialization of the input data.
@param inputData: the variable, solve-specific input data
@type inputData: OPLCollector
@param solutionId: an identifier for the solution, used in iterative algorithms (set to empty string if not needed)
@type solutionId: String
@return: a solution collector
'''
inputs = []
if self.model is None:
raise ValueError("A model attachment must be provided to the optimizer")
if self.model: # is not empty
stream = self.model.toStream()
inputs.append({"name": self.model.getName(), "file": stream})
self.streamsRegistry.append(stream)
if self.attachments: # is not empty
for f in self.attachments:
stream = urllib.FancyURLopener(self.attachments[f])
inputs.append({"name": f, "file": stream})
self.streamsRegistry.append(stream)
if inputData is not None:
outStream = cStringIO.StringIO()
inputData.setJsonDestination(outStream).toJSON()
inStream = cStringIO.StringIO(outStream.getvalue())
inputs.append({"name": inputData.getName() + ".json", "file": inStream})
self.streamsRegistry.extend([outStream, inStream])
response = self.jobclient.execute(
input=inputs,
output="results.json",
load_solution=True,
log="solver.log",
gzip=True,
waittime=300, # seconds
delete_on_completion=False)
self.jobid = response.jobid
status = self.jobclient.get_execution_status(self.jobid)
if status == JobExecutionStatus.PROCESSED:
results = cStringIO.StringIO(response.solution)
self.streamsRegistry.append(results)
self.solveStatus = response.job_info.get(
'solveStatus') # INFEASIBLE_SOLUTION or UNBOUNDED_SOLUTION or OPTIMAL_SOLUTION or...
solution = (OPLCollector(self.getName() + "Result" + solutionId, self.resultDataModel)).setJsonSource(
results).fromJSON()
self.history.append(solution)
elif status == JobExecutionStatus.FAILED:
# get failure message if defined
message = ""
if (response.getJob().getFailureInfo() != None):
message = response.getJob().getFailureInfo().getMessage()
print("Failed " + message)
else:
print("Job Status: " + status)
for s in self.streamsRegistry:
s.close();
self.jobclient.delete_job(self.jobid);
return solution
def getSolveStatus(self):
"""
@return the solve status as a string
Attributes:
UNKNOWN: The algorithm has no information about the solution.
FEASIBLE_SOLUTION: The algorithm found a feasible solution.
OPTIMAL_SOLUTION: The algorithm found an optimal solution.
INFEASIBLE_SOLUTION: The algorithm proved that the model is infeasible.
UNBOUNDED_SOLUTION: The algorithm proved the model unbounded.
INFEASIBLE_OR_UNBOUNDED_SOLUTION: The model is infeasible or unbounded.
"""
return self.solveStatus
