def connectServer(self): #doCloud if self.doCloudUrl != None: if import_error: self.Error(import_error) if sys.platform == "win32": os.environ['REQUESTS_CA_BUNDLE'] = os.path.join(os.path.dirname(sys.executable), "GMSPython", "Lib", "site-packages", "certifi", "cacert.pem") self.doCloudClient = JobClient(self.doCloudUrl, self.doCloudKey) #neos else: if self.logopt in [1,3,4]: sys.stdout.write("Connecting to: %s://%s:%s\n" % (self.serverProtocol,self.serverHost,self.serverPort)) if self.logopt in [2,4]: # Append the message to the logfile indicated try: f = open(kestrel.logfilename,'a') f.write("Connecting to: %s://%s:%s\n" % (self.serverProtocol,self.serverHost,self.serverPort)) f.close() except IOError as e: self.Fatal("Could not append to log file %s" % self.logfilename) self.neos = xmlrpc.client.Server("%s://%s:%s" % (self.serverProtocol,self.serverHost,self.serverPort)) reply = self.neos.ping() if reply.find('alive') < 0: raise KestrelException("Unable to contact NEOS at https://%s:%d" % \ (self.host, self.port))
def __init__(self, problemName, model=None, resultDataModel=None, credentials=None, *attachments): ''' Constructs an Optimizer instance. The instance requires an optimization model as a parameter. You can also provide one or more data files as attachments, either in OPL .dat or in JSON format. This data does not change from solve to solve. If you have input data that does change, you can provide it to the solve method as an OPLCollector object. :param problemName: name of this optimization problem instance :type problemName: String :param model: an optimization model written in OPL :type model: Model.Source object or String :param resultDataModel: the application data model for the results of the optimization :type resultDataModel: dict<String, StructType> :param credentials: DOcplexcloud url and api key :type credentials: {"url":String, "key":String} :param attachments: URLs for files representing the data that does not vary from solve to solve :type attachments: list<URL> ''' self.name = problemName self.model = model self.resultDataModel = resultDataModel self.attachData(attachments) self.streamsRegistry = [] self.history = [] self.credentials = credentials self.jobclient = JobClient(credentials["url"], credentials["key"]); self.solveStatus = JobSolveStatus.UNKNOWN;
def run_model(): url = 'https://api-oaas.docloud.ibmcloud.com/job_manager/rest/v1/' key = 'api_fc2d8028-3ebc-4a3a-8664-b4018b1c05a8' client = JobClient(url=url, api_key=key) resp = client.execute(input=[ 'workshop_model.py', '../input/best_submission.csv', '../input/family_data.csv' ], output='solution.json', load_solution=True, log='logs.txt')
def __init__(self, url, api_key, concurrent_jobs): """ Creates a new controller to submit job asynchronously. Args: url: The DOcloud url. api_key: The DOcloud api key. concurrent_jobs: The number of concurrently submitted jobs. """ self.nb_threads = concurrent_jobs self.client = JobClient(url, api_key) # This is the opl model file self.mod_file = "models/truck.mod" # The executor self.executor = ThreadPoolExecutor(self.nb_threads)
def run_command(prog, argv, url=None, key=None): description = '''Command line client for DOcplexcloud.''' epilog = '''Command details: info Get and display information for the jobs which ids are specified as ARG. download Download the attachment to the the current directory. rm Delete the jobs which ids are specfied as ARG. rm all Delete all jobs. logs Download and display the logs for the jobs which id are specified. ls Lists the jobs.''' epilog_cli = ''' execute Submit a job and wait for end of execution. Each ARG that is a file is uploaded as the job input. Example: Example: python run.py execute model.py model.data -v executes a job which input files are model.py and model.dada, in verbose mode. ''' filter_help = ''' Within filters, the following variables are defined: now: current date and time as timestamp in millisec minute: 60 sec in millisec hour: 60 minutes in millisec day: 24 hour in millisec job: The current job being filtered Example filter usage: Delete all jobs older than 3 hour python -m docplex.cli --filter "now-job['startedAt'] > 3*hour " rm ''' if ip is None: epilog += epilog_cli epilog += filter_help parser = argparse.ArgumentParser(prog=prog, description=description, epilog=epilog, formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument('command', metavar='COMMAND', help='DOcplexcloud command') parser.add_argument('arguments', metavar='ARG', nargs='*', help='Arguments for the command') parser.add_argument('--no-delete', action='store_true', default=False, dest='nodelete', help="If specified, jobs are not deleted after execution") parser.add_argument('-v', '--verbose', action='store_true', help='Verbose mode') parser.add_argument('--as', nargs=1, metavar='HOST', dest="host_config", default=None, help="'as host' - use the cplex_config_<HOST>.py configuration file found in PYTHONPATH") parser.add_argument('--url', nargs=1, metavar='URL', dest="url", default=None, help="The DOcplexcloud connection URL. If not specified, will use those found in docplex config files") parser.add_argument('--key', nargs=1, metavar='API_KEY', dest="key", default=None, help="The DOcplexcloud connection key. If not specified, will use those found in docplex config files") parser.add_argument('--details', action='store_true', default=False, help='Display solve details as they are available') parser.add_argument('--filter', metavar='FILTER', default=None, help='filter on job. Example: --filter "True if (now-job.createdAt) > 3600"') parser.add_argument('--quiet', '-q', action='store_true', default=False, help='Only show numeric IDs as output') args = parser.parse_args(argv) program_result = ProgramResults() # Get the context here so that we have some credentials at hand context = Context.make_default_context() if args.host_config is not None: config_name = "cplex_config_%s.py" % args.host_config[0] config_file = list(filter(os.path.isfile, [os.path.join(x, config_name) for x in sys.path])) if len(config_file) == 0: print("Could not find config file for host: %s" % args.host_config[0]) program_result.return_code = -1 return(program_result) if args.verbose: print("Overriding host config with: %s" % config_file[0]) context.read_settings(config_file[0]) # use credentials in context unless they are given to this function client_url = context.solver.docloud.url if url is None else url client_key = context.solver.docloud.key if key is None else key # but if there are some credentials in arguments (--url, --key), use them if args.url: client_url = args.url if args.key: client_key = args.key if args.verbose: print('**** Connecting to %s with key %s' % (client_url, client_key)) print('Will send command %s' % args.command) print('Arguments:') for i in args.arguments: print(' -> %s' % i) print('verbose = %s' % args.verbose) client = JobClient(client_url, client_key) target_jobs = [] if args.filter: jobs = client.get_all_jobs() now = (datetime.datetime.now() - datetime.datetime(1970,1,1)).total_seconds() * 1000.0 minute = 60 * 1000 hour = 60 * minute day = 24 * hour context = {'now': now, 'minute': minute, 'hour': hour, 'day': day, } for j in jobs: context['job'] = j keep = False try: keep = eval(args.filter, globals(), context) except KeyError: # if a key was not foud, just assume expression is false keep = False if keep: target_jobs.append(j) if target_jobs: for i in target_jobs: print('applying to %s' % i['_id']) if args.command == 'ls': ls_jobs(client, program_result, quiet=args.quiet, selected_jobs=target_jobs) elif args.command == 'info': if target_jobs: args.arguments = [x["_id"] for x in target_jobs] elif len(args.arguments) == 1 and args.arguments[0] == 'all': args.arguments = [x["_id"] for x in client.get_all_jobs()] for id in args.arguments: info_text = "NOT FOUND" try: job = client.get_job(id) info_text = json.dumps(job, indent=3) except: pass print("%s:\n%s" % (id, info_text)) elif args.command == 'rm': if target_jobs: joblist = [x["_id"] for x in target_jobs] elif args.arguments: joblist = args.arguments else: joblist = shlex.split(sys.stdin.read()) rm_job(client, joblist, verbose=args.verbose) elif args.command == 'logs': if target_jobs: if len(target_jobs) != 1: print('Logs can only be retrieved when filter select one job (actual selection count = %s)' % len(target_jobs)) program_result.return_code = -1 return(program_result) args.arguments = [x["_id"] for x in target_jobs] if not args.arguments: print('Please specify job list in arguments or using filter.') program_result.return_code = -1 return(program_result) for jid in args.arguments: log_items = client.get_log_items(jid) for log in log_items: for record in log["records"]: print(record["message"]) elif args.command == 'download': if target_jobs: if len(target_jobs) != 1: print('Jobs can only be downloaded when filter select one job (actual selection count = %s)' % len(target_jobs)) program_result.return_code = -1 return(program_result) args.arguments = [x["_id"] for x in target_jobs] for jid in args.arguments: job = client.get_job(jid) for attachment in job['attachments']: print('downloading %s' % attachment['name']) with open(attachment['name'], 'wb') as f: f.write(client.download_job_attachment(id, attachment['name'])) elif args.command == 'execute': if target_jobs: print('Execute command does not support job filtering') program_result.return_code = -1 return(program_result) inputs = [{'name': basename(a), 'filename': a} for a in args.arguments] if args.verbose: for i in inputs: print("Uploading %s as attachment name %s" % (i['filename'], i['name'])) execute_job(client, inputs, args.verbose, args.details, args.nodelete) else: print("Unknown command: %s" % args.command) program_result.return_code = -1 return(program_result) return(program_result)
def solving_placement_problem_from_file(topology_graph, request_graph, test_num, CPLEX_PATH, cplex_models_path, results_path, locally): # Reading networkx file G_topology = read_json_file(topology_graph) G_request = read_json_file(request_graph) set_PM = list(G_topology.nodes) set_state_or_nf = list(G_request.nodes) set_state, set_nf, set_replica = [], [], [] for i in set_state_or_nf: if "function" in i: set_nf.append(i) elif "state" in i: set_state.append(i) elif "replica" in i: set_replica.append(i) if not os.path.isfile("{}/p5_cplex_model_{}_2.lp".format( cplex_models_path, test_num)): cplex_f = open('{}/p5_cplex_model_{}_2.lp'.format( cplex_models_path, test_num), mode='a') # TODO: Validating request graph for i in set_state: try: G_request.nodes[i]['size'] except: RuntimeError( "The given request graph is incorrect: State {} has no 'size' value" .format(i)) s = {i: G_request.nodes[i]['size'] for i in set_state + set_replica} c = {i: G_topology.nodes[i]['capacity'] for i in set_PM} print("Generating delay matrix...") d = generating_delay_matrix(G_topology) print("Generating state-function adjacency matrix...") e_r = generating_req_adj(set_state, set_nf + set_replica, G_request) print("Generating Function mapping matrix...") M = generating_nf_mapping_matrix(G_topology) print("Generating Anti-Affinity set") AA = generating_AA(set_state, G_request) print("Generating OR-Link set") OL = generating_OL(set_state, set_nf, set_replica, G_request) # ## Into File ############################################################################################ cplex_f = open('{}/p5_cplex_model_{}_2.lp'.format( cplex_models_path, test_num), mode='a') cplex_f.write("Minimize\n obj: [ ") servers = [i for i in set_PM if "server" in i] server_permutations = list(itertools.permutations(servers, 2)) first = True for i, j in server_permutations: asd = list( itertools.permutations(set_state + set_replica + set_nf, 2)) for u, v in asd: if (e_r[u, v] * d[i, j] * 2 > 0): if first: cplex_f.write( " {} y_({},{})_({},{})*z_({},{})\n".format( e_r[u, v] * d[i, j] * 2, i, u, j, v, u, v)) first = False else: cplex_f.write( " + {} y_({},{})_({},{})*z_({},{})\n".format( e_r[u, v] * d[i, j] * 2, i, u, j, v, u, v)) cplex_f.close() cplex_f = open('{}/p5_cplex_model_{}_2.lp'.format( cplex_models_path, test_num), mode='rb+') cplex_f.seek(-1, os.SEEK_END) cplex_f.truncate() cplex_f = open('{}/p5_cplex_model_{}_2.lp'.format( cplex_models_path, test_num), mode='a') cplex_f.write("]/2 \n") # contraint 1 -------------------------------------------------------------------------------------------- print("Generating mapping constraints") cplex_f.write("\nSubject To \n") for u in set_state + set_replica: c_name = "c1_{}".format(u) cplex_f.write(" {}: ".format(c_name)) for i in set_PM: cplex_f.write(" x_({},{}) +".format(i, u)) cplex_f.close() cplex_f = open('{}/p5_cplex_model_{}_2.lp'.format( cplex_models_path, test_num), mode='rb+') cplex_f.seek(-2, os.SEEK_END) cplex_f.truncate() cplex_f = open('{}/p5_cplex_model_{}_2.lp'.format( cplex_models_path, test_num), mode='a') cplex_f.write(" = 1\n") # contraint 2 -------------------------------------------------------------------------------------------- print("Generating capacity constraints") for i in set_PM: c_name = "c2_{}".format(i) cplex_f.write(" {}: ".format(c_name)) for u in set_state + set_replica: cplex_f.write("{} x_({},{}) +".format(s[u], i, u)) cplex_f.close() cplex_f = open('{}/p5_cplex_model_{}_2.lp'.format( cplex_models_path, test_num), mode='rb+') cplex_f.seek(-2, os.SEEK_END) cplex_f.truncate() cplex_f = open('{}/p5_cplex_model_{}_2.lp'.format( cplex_models_path, test_num), mode='a') cplex_f.write(" <= {}\n".format(c[i])) # contraint 3 -------------------------------------------------------------------------------------------- print("Generating AA constraints") for i in set_PM: if "server" in i: for u, v in AA: c_name = "c3_{}_{}_in_{}".format(u, v, i) cplex_f.write(" {}: ".format(c_name)) cplex_f.write(" x_({},{}) + x_({}, {}) <= 1\n".format( i, u, i, v)) # contraint 4 -------------------------------------------------------------------------------------------- print("Generating NF mapping constraints") for function in set_nf: for server in set_PM: c_name = "c4_{}_in_{}".format(function, server) try: if M[function] == server: cplex_f.write(" {}: ".format(c_name)) cplex_f.write(" x_({},{}) = 1\n".format( server, function)) else: cplex_f.write(" {}: ".format(c_name)) cplex_f.write(" x_({},{}) = 0\n".format( server, function)) except: cplex_f.write(" {}: ".format(c_name)) cplex_f.write(" x_({},{}) = 0\n".format(server, function)) # contraint 5 -------------------------------------------------------------------------------------------- def for_multiprocessing(list, from_, to_, test_num, process_id, cplex_models_path): print("Starting process {}, from: {}, to:{}".format( process_id, from_, to_)) c5_f = open('{}/c5_testnum{}_tmp{}_2.txt'.format( cplex_models_path, test_num, process_id), mode='a') start = from_ for i, j in list[from_:to_]: if (start % 10000) == 0: print("{}: {}".format(process_id, start)) c_name = "c5_({},{})_({},{})_0".format(i[0], i[1], j[0], j[1]) c5_f.write(" {}: ".format(c_name)) c5_f.write(" y_({},{})_({},{}) >= 0 \n".format( i[0], i[1], j[0], j[1])) c_name = "c5_({},{})_({},{})_1".format(i[0], i[1], j[0], j[1]) c5_f.write(" {}: ".format(c_name)) c5_f.write( " y_({},{})_({},{}) - x_({},{}) - x_({},{}) >= -1 \n". format(i[0], i[1], j[0], j[1], i[0], i[1], j[0], j[1])) start += 1 c5_f.close() print("Ending process {}".format(process_id)) print(datetime.datetime.now()) print("Generating QP -> ILP transformation constraints") index_set = set() for i in set_PM: for u in set_state + set_replica + set_nf: index_set.add((i, u)) index_combinations = list(itertools.permutations(index_set, 2)) index_combinations_size = len(index_combinations) print("Size of this contraints: {}".format(index_combinations_size)) from_to_list = [] core_num = 10 core_job_count = len(index_combinations) / 10 for i in range(core_num): from_ = i * core_job_count to_ = (i + 1) * core_job_count if i == core_num - 1: to_ = index_combinations_size from_to_list.append((from_, to_)) import multiprocessing processes = [] for i in range(0, core_num): p = multiprocessing.Process( target=for_multiprocessing, args=(index_combinations, from_to_list[i][0], from_to_list[i][1], test_num, i, cplex_models_path)) processes.append(p) p.start() for process in processes: process.join() tempfiles = [ "{}/c5_testnum{}_tmp{}_2.txt".format(cplex_models_path, test_num, i) for i in range(core_num) ] for tempfile in tempfiles: #print("Adding file {}".format(tempfile)) tmp_f = open(tempfile, "r") cplex_f.write(tmp_f.read()) for tempfile in tempfiles: os.remove(tempfile) # contraint 6 -------------------------------------------------------------------------------------------- print("Generating 'Does it matter?' constraints") for u in (set_state + set_nf + set_replica): for v in (set_state + set_nf + set_replica): if u != v: if is_OL(u, v, OL): c_name = "c6_({},{})_0".format(u, v) cplex_f.write("\n {}: ".format(c_name)) first = True for i, j in get_OLs(u, v, OL): if first: cplex_f.write(" z_({},{})".format(i, j)) first = False else: cplex_f.write(" + z_({},{})".format(i, j)) cplex_f.write(" = 1 \n") else: c_name = "c6_({},{})_1".format(u, v) if "function" in u and "replica" in v: cplex_f.write("\n {}: ".format(c_name)) cplex_f.write(" z_({},{}) = 0".format(u, v)) elif "replica" in u and "state" in v: cplex_f.write("\n {}: ".format(c_name)) cplex_f.write(" z_({},{}) = 0".format(u, v)) else: cplex_f.write("\n {}: ".format(c_name)) cplex_f.write(" z_({},{}) = 1".format(u, v)) # Bounds -------------------------------------------------------------------------------------------- cplex_f.write("\nBounds\n") for i in set_PM: for u in set_state + set_replica + set_nf: cplex_f.write("0 <= x_({},{}) <= 1\n".format(i, u)) index_set = set() for i in set_PM: for u in set_state + set_replica + set_nf: index_set.add((i, u)) index_permutations = list(itertools.permutations(index_set, 2)) for i, j in index_permutations: cplex_f.write("0 <= y_({},{})_({},{}) <= 1\n".format( i[0], i[1], j[0], j[1])) index_permutations = list( itertools.permutations((set_state + set_nf + set_replica), 2)) for u, v in index_permutations: cplex_f.write("0 <= z_({},{}) <= 1\n".format(u, v)) # Binaries -------------------------------------------------------------------------------------------- cplex_f.write("\nBinaries\n") for i in set_PM: for u in set_state + set_replica + set_nf: cplex_f.write(" x_({},{})\n".format(i, u)) index_set = set() for i in set_PM: for u in set_state + set_replica + set_nf: index_set.add((i, u)) index_permutations = list(itertools.permutations(index_set, 2)) for i, j in index_permutations: cplex_f.write(" y_({},{})_({},{})\n".format( i[0], i[1], j[0], j[1])) index_permutations = list( itertools.permutations((set_state + set_nf + set_replica), 2)) for u, v in index_permutations: cplex_f.write(" z_({},{})\n".format(u, v)) time.sleep(2) cplex_f.write("End\n") cplex_f.close() ######################################################################################################## if not os.path.isfile(CPLEX_PATH): raise RuntimeError('CPLEX does not exist ({})'.format(CPLEX_PATH)) if not os.path.isfile("{}/p5_cplex_model_{}_2.sav".format( cplex_models_path, test_num)): subprocess.call( "{} -c 'read {}/p5_cplex_model_{}_2.lp' 'write {}/p5_cplex_model_{}_2.sav sav'" .format(CPLEX_PATH, cplex_models_path, test_num, cplex_models_path, test_num), shell=True) t1 = datetime.datetime.now() cost = 0 mapping_result = {i: "" for i in set_state + set_nf + set_replica} if locally: # solving problem in locally print("\n\nSolving the problem locally - 2") subprocess.call( "{} -c 'read {}/p5_cplex_model_{}_2.mps' 'optimize' 'write {}/p5_cplex_result_{}_2 sol'" .format(CPLEX_PATH, cplex_models_path, test_num, results_path, test_num), shell=True) else: print("\n\nSolving the problem remotely in the IBM cloud - 2") if not os.path.isfile("{}/p5_cplex_result_{}_2".format( results_path, test_num)): client = JobClient( "https://api-oaas.docloud.ibmcloud.com/job_manager/rest/v1/", "api_e7f3ec88-92fd-4432-84d7-f708c4a33132") print( "You can check the status of the problem procesing here: https://dropsolve-oaas.docloud.ibmcloud.com/dropsolve" ) resp = client.execute(input=[ "{}/p5_cplex_model_{}_2.sav".format(cplex_models_path, test_num) ], output="{}/p5_cplex_result_{}_2".format( results_path, test_num)) if resp.job_info["solveStatus"] == "INFEASIBLE_SOLUTION": print("There is no valid mapping!") return 0 def is_json_file(report_input_file): with open(report_input_file) as unknown_file: c = unknown_file.read(1) if c != '<': return True return False t2 = datetime.datetime.now() if is_json_file("{}/p5_cplex_result_{}_2".format(results_path, test_num)): with open("{}/p5_cplex_result_{}_2".format(results_path, test_num)) as f: result = json.load(f) for i in result["CPLEXSolution"]["variables"]: if ("x_" in list(i["name"])) and i["value"] == str(1): # print("{} = 1".format(i["name"])) server = i["name"].split(',')[0][3:] ve = i["name"].split(',')[1][:-1] mapping_result[ve] = server print("*** Delay cost: {} ***".format( result["CPLEXSolution"]["header"]["objectiveValue"])) cost = result["CPLEXSolution"]["header"]["objectiveValue"] else: with open("{}/p5_cplex_result_{}_2".format(results_path, test_num), 'r') as file: xml_result = file.read().replace('\n', '') result = xmltodict.parse(xml_result) print("*** Delay cost: {} ***".format( result["CPLEXSolution"]["header"]["@objectiveValue"])) cost = result["CPLEXSolution"]["header"]["@objectiveValue"] running_time = t2 - t1 print("RUNNING TIME: {}".format(running_time)) return cost, mapping_result, running_time
def submit_model_data(self, attachments=None, gzip=False, info_callback=None, info_to_monitor=None): """Submits a job to the cloud service. Args: attachments: A list of attachments. Each attachement is a dict with the following keys: - 'name' : the name of the attachment - 'data' : the data for the attachment gzip: If ``True``, data is gzipped before sent over the network info_callback: A call back to be called when some info are available. That callback takes one parameter that is a dict containing the info as they are available. info_to_monitor: A set of information to monitor with info_callback. Currently, can be ``jobid`` and ``progress``. """ self.__vars = None self.timed_out = False self.results.clear() if not info_to_monitor: info_to_monitor = {} # check that url is valid parts = urlparse(self.docloud_context.url) if not parts.scheme: raise DOcloudConnectorException( "Malformed URL: '%s': No schema supplied." % self.docloud_context.url) proxies = self.docloud_context.proxies try: client = JobClient(self.docloud_context.url, self.docloud_context.key, proxies=proxies) except TypeError: # docloud client <= 1.0.172 do not have the proxes warnings.warn( "Using a docloud client that do not support warnings in init()", UserWarning) client = JobClient(self.docloud_context.url, self.docloud_context.key) self.log("client created") if proxies: self.log("proxies = %s" % proxies) # prepare client if self.docloud_context.log_requests: client.rest_callback = \ lambda m, u, *a, **kw: self._rest_callback(m, u, *a, **kw) client.verify = self.docloud_context.verify client.timeout = self.docloud_context.get('timeout', None) try: try: # Extract the list of attachment names att_names = [a['name'] for a in attachments] # create job jobid = client.create_job( attachments=att_names, parameters=self.docloud_context.job_parameters) self.log("job creation submitted, id is: {0!s}".format(jobid)) if info_callback and 'jobid' in info_to_monitor: info_callback({'jobid': jobid}) except ConnectionError as c_e: raise DOcloudConnectorException( "Cannot connect to {0}, error: {1}".format( self.docloud_context.url, str(c_e))) try: # now upload data for a in attachments: pos = 0 if 'data' in a: att_data = {'data': a['data']} elif 'file' in a: att_data = {'file': a['file']} pos = a['file'].tell() elif 'filename' in a: att_data = {'filename': a['filename']} client.upload_job_attachment(jobid, attid=a['name'], **att_data) self.log("Attachment: %s has been uploaded" % a['name']) if self.docloud_context.debug_dump_dir: target_dir = self.docloud_context.debug_dump_dir if not os.path.exists(target_dir): os.makedirs(target_dir) self.log("Dumping input attachment %s to dir %s" % (a['name'], target_dir)) with open(os.path.join(target_dir, a['name']), "wb") as f: if 'data' in 'a': if isinstance(a['data'], bytes): f.write(a['data']) else: f.write(a['data'].encode('utf-8')) else: a['file'].seek(pos) f.write(a['file']) # execute job client.execute_job(jobid) self.log("DOcplexcloud execute submitted has been started") # get job execution status until it's processed or failed timedout = False try: self._executionStatus = self.wait_for_completion( client, jobid, info_callback=info_callback, info_to_monitor=info_to_monitor) except DOcloudInterruptedException: timedout = True self.log("docloud execution has finished") # get job status. Do this before any time out handling self.jobInfo = client.get_job(jobid) if self.docloud_context.fire_last_progress and info_callback: progress_data = self.map_job_info_to_progress_data( self.jobInfo) info_callback({'progress': progress_data}) if timedout: self.timed_out = True self.log("Solve timed out after {waittime} sec".format( waittime=self.docloud_context.waittime)) return # get solution => download all attachments try: for a in client.get_job_attachments(jobid): if a['type'] == 'OUTPUT_ATTACHMENT': name = a['name'] self.log("Downloading attachment '%s'" % name) attachment_as_string = self._as_string( client.download_job_attachment(jobid, attid=name)) self.results[name] = attachment_as_string if self.docloud_context.debug_dump_dir: target_dir = self.docloud_context.debug_dump_dir if not os.path.exists(target_dir): os.makedirs(target_dir) self.log("Dumping attachment %s to dir %s" % (name, target_dir)) with open(os.path.join(target_dir, name), "wb") as f: f.write( attachment_as_string.encode('utf-8')) except DOcloudNotFoundError: self.log("no solution in attachment") self.log("docloud results have been received") # on_solve_finished_cb if self.docloud_context.on_solve_finished_cb: self.docloud_context.on_solve_finished_cb(jobid=jobid, client=client, connector=self) return finally: if self.docloud_context.delete_job: deleted = client.delete_job(jobid) self.log("delete status for job: {0!s} = {1!s}".format( jobid, deleted)) finally: client.close()
from docloud.job import JobClient if __name__ == '__main__': url = "Paste your base URL" api_key = "Paste your api key" client = JobClient(url, api_key) resp = client.execute(input=["models/truck.dat", "models/truck.mod"], output="results.json")
def solving_placement_problem_from_file(topology_graph, request_graph, test_num): if not os.path.isfile( "./cplex_models/p5_cplex_model_{}.lp".format(test_num)): cplex_f = open('./cplex_models/p5_cplex_model_{}.lp'.format(test_num), mode='a') # Reading networkx file G_topology = read_json_file(topology_graph) G_request = read_json_file(request_graph) set_PM = list(G_topology.nodes) set_state_or_nf = list(G_request.nodes) set_state, set_nf, set_replica = [], [], [] for i in set_state_or_nf: if "function" in i: set_nf.append(i) elif "state" in i: set_state.append(i) elif "replica" in i: set_replica.append(i) # TODO: Validating request graph for i in set_state: try: G_request.nodes[i]['size'] except: RuntimeError( "The given request graph is incorrect: State {} has no 'size' value" .format(i)) s = {i: G_request.nodes[i]['size'] for i in set_state + set_replica} c = {i: G_topology.nodes[i]['capacity'] for i in set_PM} print("Generating delay matrix...") d = generating_delay_matrix(G_topology) print("Generating state-function adjacency matrix...") e_r = generating_req_adj(set_state, set_nf + set_replica, G_request) print("Generating Function mapping matrix...") M = generating_nf_mapping_matrix(G_topology) print("Generating Anti-Affinity set") AA = generating_AA(set_state, G_request) print("Generating OR-Link set") OL = generating_OL(set_state, set_nf, set_replica, G_request) opt_model = cpx.Model(name="P5") # Binary variables print("Creating variables 1...") x_vars = {(i, u): opt_model.binary_var(name="x_({0},{1})".format(i, u)) for i in set_PM for u in set_state + set_replica + set_nf} print("Creating variables 2...") index_set = set() for i in set_PM: for u in set_state + set_replica + set_nf: index_set.add((i, u)) index_permutations = list(itertools.permutations(index_set, 2)) y_vars = {(i[0], i[1], j[0], j[1]): opt_model.binary_var( name="y_({},{})_({},{})".format(i[0], i[1], j[0], j[1])) for i, j in index_permutations} print("Creating variables 3...") index_permutations = list( itertools.permutations((set_state + set_nf + set_replica), 2)) z_vars = {(u, v): opt_model.binary_var(name="z_({},{})".format(u, v)) for u, v in index_permutations} # == constraints 1 - virtual element can be mapped into only one server print( "Creating constraints 1 - virtual element can be mapped into only one server" ) for u in set_state + set_replica: c_name = "c1_{}".format(u) opt_model.add_constraint(ct=opt_model.sum(x_vars[i, u] for i in set_PM) == 1, ctname=c_name) # <= constraints 2 - server capacity constraint print("Creating constraints 2 - server capacity constraint") for i in set_PM: c_name = "c2_{}".format(i) opt_model.add_constraint( ct=opt_model.sum(s[u] * x_vars[i, u] for u in set_state + set_replica) <= c[i], ctname=c_name) # <= constraints 3 - anti-affinity rules print("Creating constraints 3 - anti-affinity rules") for i in set_PM: if "server" in i: for u, v in AA: c_name = "c3_{}_{}_in_{}".format(u, v, i) opt_model.add_constraint( ct=(x_vars[i, u] + x_vars[i, v]) <= 1, ctname=c_name) # == constraints 4 - NFs running places print("Creating constraints 4 - NFs running places") for function in set_nf: for server in set_PM: c_name = "c4_{}_in_{}".format(function, server) try: if M[function] == server: opt_model.add_constraint(ct=x_vars[server, function] == 1, ctname=c_name) # print("x_vars[{}, {}] == 1".format(server, function)) else: opt_model.add_constraint(ct=x_vars[server, function] == 0, ctname=c_name) # print("x_vars[{}, {}] == 0".format(server, function)) except: opt_model.add_constraint(ct=x_vars[server, function] == 0, ctname=c_name) # print("x_vars[{}, {}] == 0".format(server, function)) # >= constraints 5 - QP -> ILP transformation constraints print("Creating constraints 5 - QP -> ILP transformation constraints") index_set = set() for i in set_PM: for u in set_state + set_replica + set_nf: index_set.add((i, u)) index_combinations = list(itertools.permutations(index_set, 2)) for i, j in index_combinations: c_name = "c5_({},{})_({},{})_0".format(i[0], i[1], j[0], j[1]) opt_model.add_constraint(ct=y_vars[i[0], i[1], j[0], j[1]] >= 0, ctname=c_name) c_name = "c5_({},{})_({},{})_1".format(i[0], i[1], j[0], j[1]) opt_model.add_constraint( ct=y_vars[i[0], i[1], j[0], j[1]] >= (x_vars[i[0], i[1]] + x_vars[j[0], j[1]] - 1), ctname=c_name) print("Creating constraints 6 - z variable rules") for u in (set_state + set_nf + set_replica): for v in (set_state + set_nf + set_replica): if u != v: if is_OL(u, v, OL): c_name = "c6_({},{})_0".format(u, v) opt_model.add_constraint(ct=opt_model.sum( z_vars[(i, j)] for i, j in get_OLs(u, v, OL)) == 1, ctname=c_name) else: c_name = "c6_({},{})_1".format(u, v) if "function" in u and "replica" in v: opt_model.add_constraint(ct=z_vars[(u, v)] == 0, ctname=c_name) elif "replica" in u and "state" in v: opt_model.add_constraint(ct=z_vars[(u, v)] == 0, ctname=c_name) else: opt_model.add_constraint(ct=z_vars[(u, v)] == 1, ctname=c_name) print("Creating Objective function...") servers = [i for i in set_PM if "server" in i] server_permutations = list(itertools.permutations(servers, 2)) objective = opt_model.sum( y_vars[i, u, j, v] * e_r[u, v] * d[i, j] * z_vars[u, v] for i, j in server_permutations for u, v in list( itertools.permutations(set_state + set_replica + set_nf, 2))) # # # for minimization opt_model.minimize(objective) print("Exporting the problem") opt_model.export_as_lp(basename="p5_cplex_model_{}".format(test_num), path="./cplex_models") subprocess.call( "/home/epmetra/projects/cplex/cplex/bin/x86-64_linux/cplex -c 'read /home/epmetra/projects/LO/cplex_models/p5_cplex_model_{}.lp' 'write /home/epmetra/projects/LO/cplex_models/p5_cplex_model_{}.mps mps'" .format(test_num, test_num), shell=True) # solving with local cplex # print("Solving the problem locally") # print(datetime.datetime.now()) # asd = opt_model.solve() # solving in the docplex cloud print("Solving the problem by the cloud - 1") print(datetime.datetime.now()) if not os.path.isfile( "optimization_results/p5_cplex_result_{}.json".format(test_num)): client = JobClient( "https://api-oaas.docloud.ibmcloud.com/job_manager/rest/v1/", "api_e7f3ec88-92fd-4432-84d7-f708c4a33132") print( "You can check the status of the problem procesing here: https://dropsolve-oaas.docloud.ibmcloud.com/dropsolve" ) resp = client.execute( input=["./cplex_models/p5_cplex_model_{}.mps".format(test_num)], output="optimization_results/p5_cplex_result_{}.json".format( test_num)) mapping_result = {i: "" for i in set_state + set_nf + set_replica} if resp.job_info["solveStatus"] == "INFEASIBLE_SOLUTION": print("There is no valid mapping!") return 0 else: with open("./optimization_results/p5_cplex_result_{}.json".format( test_num)) as f: result = json.load(f) for i in result["CPLEXSolution"]["variables"]: if ("x_" in i["name"]) and i["value"] == str(1): print("{} = 1".format(i["name"])) server = i["name"].split(',')[0][3:] ve = i["name"].split(',')[1][:-1] mapping_result[ve] = server print("*** Delay cost: {} ***".format( result["CPLEXSolution"]["header"]["objectiveValue"])) return result["CPLEXSolution"]["header"][ "objectiveValue"], mapping_result else: with open("./optimization_results/p5_cplex_result_{}.json".format( test_num)) as f: result = json.load(f) for i in result["CPLEXSolution"]["variables"]: if ("x_" in i["name"]) and i["value"] == str(1): print("{} = 1".format(i["name"])) print("*** Delay cost: {} ***".format( result["CPLEXSolution"]["header"]["objectiveValue"])) return result["CPLEXSolution"]["header"]["objectiveValue"]
solver_manager = SolverManagerFactory('neos') opt = SolverFactory('cplex', solver_io='lp') results = solver_manager.solve(model, opt=opt) model.solutions.store_to(results) print(results) # >>>> 3. docplexcloud # key = #DOCPLEX API KEY (get from docloud website) # base_url = #DOCPLEX URL (get from docloud website) # Dependencies : docloud (pip install docloud) import json import glob import pandas from docloud.job import JobClient client = JobClient(base_url, key) model.write("temp.lp", io_options={"symbolic_solver_labels": True}) # with open("temp.lp") as lpfile: # resp = client.execute(input=lpfile,output=None,load_solution = True) file = glob.glob("temp.lp") resp = client.execute(input=file, output=None, load_solution=True) solution = json.loads(resp.solution.decode("utf-8")) #os.remove(file) for i, k in solution['CPLEXSolution']['header'].items(): print(i, ':', k) results = pandas.DataFrame(solution['CPLEXSolution']['variables']).filter( items=['index', 'name', 'value', 'status']) print(results) # >>>> 4. GUROBI
def run(choice, threshold, confidence_interval, facility_number, min_threshold): """ This solves the corresponding optimization problem to cplex cloud. It takes 5 parameters: i) choice : An integer , (1-5) which represents the optimization model choice of the user ii) threshold(m) : An integer , which represents the maximum possible distance between two districts which are counted as appropriate_pairs , meter iii) confidence_interval : An integer , (0-100) which represents the confidence level of the stochastic model iv) facility_number : An integer , (0 - NUMBER_OF_DISTRICT) which represents the maximum fire station number for maximum coverage models v) min_threshold(min) : An integer , which represents the maximum traveling time between two districts in terms of minutes. It returns nothing but it creates a .txt file that contains the solution of the corresponding optimization problem. """ #Create a clinet client = JobClient(CPLEX_BASE_URL, CPLEX_API_KEY) # Choice == 1 , base model if choice == 1: resp = client.execute(input=[ "Mod_Files/BaseModel.mod", "Mod_Files/BaseModel_" + str(threshold) + ".dat" ], output="Solutions/BaseModel_Sol_" + str(threshold) + ".txt") return "Solutions/BaseModel_Sol_" + str(threshold) + ".txt" # Choice == 2 , max_coverage model if choice == 2: resp = client.execute(input=[ "Mod_Files/MultiCoverage.mod", "Mod_Files/MultiCoverage_" + str(threshold) + ".dat" ], output="Solutions/MultiCoverage_Sol_" + str(threshold) + ".txt") return "Solutions/MultiCoverage_Sol_" + str(threshold) + ".txt" # Choice == 3 , max coverage if choice == 3: resp = client.execute(input=[ "Mod_Files/MaxCoverage.mod", "Mod_Files/MaxCoverage_" + str(threshold) + "_" + str(facility_number) + ".dat" ], output="Solutions/MaxCoverage_Sol_" + str(threshold) + "_" + str(facility_number) + ".txt") return "Solutions/MaxCoverage_Sol_" + str(threshold) + "_" + str( facility_number) + ".txt" # Choice == 4 , stochastic_coverage if choice == 4: resp = client.execute(input=[ "Mod_Files/StochasticCoverage.mod", "Mod_Files/Stochastic_Coverage_" + str(min_threshold) + "_" + str(confidence_interval) + ".dat" ], output="Solutions/Stochastic_Coverage_Sol_" + str(min_threshold) + "_" + str(confidence_interval) + ".txt") return "Solutions/Stochastic_Coverage_Sol_" + str( min_threshold) + "_" + str(confidence_interval) + ".txt" # Choice == 5 , stochastic max coverage if choice == 5: resp = client.execute(input=[ "Mod_Files/MaxCoverage.mod", "Mod_Files/Stochastic_MaxCoverage_" + str(min_threshold) + "_" + str(facility_number) + "_" + str(confidence_interval) + ".dat" ], output="Solutions/Stochastic_MaxCoverage_Sol_" + str(min_threshold) + "_" + str(facility_number) + "_" + str(confidence_interval) + ".txt") return "Solutions/Stochastic_MaxCoverage_Sol_" + str( min_threshold) + "_" + str(facility_number) + "_" + str( confidence_interval) + ".txt"