def optimize(self):
"""
If CPLEX is installed locally, we can use that to solve the problem.
Otherwise, we can use DOcplexcloud. For docloud solve, we need valid 'url' and 'key'.
Note, that if 'url' and 'key' parameters are present,
the solve will be started on DOcplexcloud even if CPLEX is available.
e.g. this forces the solve on DOcplexcloud:
model.solve(url='https://foo.com', key='bar')
Using 'docplex.mp.context.Context', it is possible to control how to solve.
"""
if model_params['write_lp']:
logger.info('Writing the lp file!')
self.model.export_as_lp('./{}.lp'.format(self.model.name))
ctx = Context()
ctx.solver.docloud.url = model_params['url']
ctx.solver.docloud.key = model_params['api_key']
agent = 'docloud' if model_params['cplex_cloud'] else 'local'
# There are several ways to set the parameters. Here are two ways:
# method 1:
if model_params['mip_gap']:
self.model.parameters.mip.tolerances.mipgap = model_params[
'mip_gap']
if model_params['time_limit']:
self.model.set_time_limit(model_params['time_limit'])
# # method 2:
# cplex_parameters = {'mip.tolerances.mipgap': model_params['mip_gap'],
# 'timelimit': model_params['time_limit']}
# ctx.update(cplex_parameters, create_missing_nodes=True)
logger.info('Optimization starts!')
if model_params['write_log']:
with open("cplex.log", "w") as outs:
# prints CPLEX output to file "cplex.log"
self.model.solve(context=ctx, agent=agent, log_output=outs)
else:
self.model.solve(context=ctx,
agent=agent,
log_output=model_params['display_log'])
if self.model.solve_details.status == 'optimal':
logger.info('The solution is optimal and the objective value '
'is ${:,.2f}!'.format(self.model.objective_value))
def run_command(prog, argv, url=None, key=None):
description = '''Command line client for DOcplexcloud.'''
epilog = '''Command details:
info Get and display information for the jobs which ids are
specified as ARG.
download Download the attachment to the the current directory.
rm Delete the jobs which ids are specfied as ARG.
rm all Delete all jobs.
logs Download and display the logs for the jobs which id are specified.
ls Lists the jobs.'''
epilog_cli = '''
execute Submit a job and wait for end of execution. Each ARG that
is a file is uploaded as the job input. Example:
Example: python run.py execute model.py model.data -v
executes a job which input files are model.py and
model.dada, in verbose mode.
'''
filter_help = '''
Within filters, the following variables are defined:
now: current date and time as timestamp in millisec
minute: 60 sec in millisec
hour: 60 minutes in millisec
day: 24 hour in millisec
job: The current job being filtered
Example filter usage:
Delete all jobs older than 3 hour
python -m docplex.cli --filter "now-job['startedAt'] > 3*hour " rm
'''
if ip is None:
epilog += epilog_cli
epilog += filter_help
parser = argparse.ArgumentParser(prog=prog, description=description, epilog=epilog,
formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument('command',
metavar='COMMAND',
help='DOcplexcloud command')
parser.add_argument('arguments', metavar='ARG', nargs='*',
help='Arguments for the command')
parser.add_argument('--no-delete', action='store_true', default=False,
dest='nodelete',
help="If specified, jobs are not deleted after execution")
parser.add_argument('-v', '--verbose', action='store_true',
help='Verbose mode')
parser.add_argument('--as', nargs=1, metavar='HOST',
dest="host_config", default=None,
help="'as host' - use the cplex_config_<HOST>.py configuration file found in PYTHONPATH")
parser.add_argument('--url', nargs=1, metavar='URL',
dest="url", default=None,
help="The DOcplexcloud connection URL. If not specified, will use those found in docplex config files")
parser.add_argument('--key', nargs=1, metavar='API_KEY',
dest="key", default=None,
help="The DOcplexcloud connection key. If not specified, will use those found in docplex config files")
parser.add_argument('--details', action='store_true', default=False,
help='Display solve details as they are available')
parser.add_argument('--filter', metavar='FILTER', default=None,
help='filter on job. Example: --filter "True if (now-job.createdAt) > 3600"')
parser.add_argument('--quiet', '-q', action='store_true', default=False,
help='Only show numeric IDs as output')
args = parser.parse_args(argv)
program_result = ProgramResults()
# Get the context here so that we have some credentials at hand
context = Context.make_default_context()
if args.host_config is not None:
config_name = "cplex_config_%s.py" % args.host_config[0]
config_file = list(filter(os.path.isfile, [os.path.join(x, config_name) for x in sys.path]))
if len(config_file) == 0:
print("Could not find config file for host: %s" % args.host_config[0])
program_result.return_code = -1
return(program_result)
if args.verbose:
print("Overriding host config with: %s" % config_file[0])
context.read_settings(config_file[0])
# use credentials in context unless they are given to this function
client_url = context.solver.docloud.url if url is None else url
client_key = context.solver.docloud.key if key is None else key
# but if there are some credentials in arguments (--url, --key), use them
if args.url:
client_url = args.url
if args.key:
client_key = args.key
if args.verbose:
print('**** Connecting to %s with key %s' % (client_url, client_key))
print('Will send command %s' % args.command)
print('Arguments:')
for i in args.arguments:
print(' -> %s' % i)
print('verbose = %s' % args.verbose)
client = JobClient(client_url, client_key)
target_jobs = []
if args.filter:
jobs = client.get_all_jobs()
now = (datetime.datetime.now() - datetime.datetime(1970,1,1)).total_seconds() * 1000.0
minute = 60 * 1000
hour = 60 * minute
day = 24 * hour
context = {'now': now,
'minute': minute,
'hour': hour,
'day': day,
}
for j in jobs:
context['job'] = j
keep = False
try:
keep = eval(args.filter, globals(), context)
except KeyError: # if a key was not foud, just assume expression is false
keep = False
if keep:
target_jobs.append(j)
if target_jobs:
for i in target_jobs:
print('applying to %s' % i['_id'])
if args.command == 'ls':
ls_jobs(client, program_result, quiet=args.quiet, selected_jobs=target_jobs)
elif args.command == 'info':
if target_jobs:
args.arguments = [x["_id"] for x in target_jobs]
elif len(args.arguments) == 1 and args.arguments[0] == 'all':
args.arguments = [x["_id"] for x in client.get_all_jobs()]
for id in args.arguments:
info_text = "NOT FOUND"
try:
job = client.get_job(id)
info_text = json.dumps(job, indent=3)
except:
pass
print("%s:\n%s" % (id, info_text))
elif args.command == 'rm':
if target_jobs:
joblist = [x["_id"] for x in target_jobs]
elif args.arguments:
joblist = args.arguments
else:
joblist = shlex.split(sys.stdin.read())
rm_job(client, joblist, verbose=args.verbose)
elif args.command == 'logs':
if target_jobs:
if len(target_jobs) != 1:
print('Logs can only be retrieved when filter select one job (actual selection count = %s)' % len(target_jobs))
program_result.return_code = -1
return(program_result)
args.arguments = [x["_id"] for x in target_jobs]
if not args.arguments:
print('Please specify job list in arguments or using filter.')
program_result.return_code = -1
return(program_result)
for jid in args.arguments:
log_items = client.get_log_items(jid)
for log in log_items:
for record in log["records"]:
print(record["message"])
elif args.command == 'download':
if target_jobs:
if len(target_jobs) != 1:
print('Jobs can only be downloaded when filter select one job (actual selection count = %s)' % len(target_jobs))
program_result.return_code = -1
return(program_result)
args.arguments = [x["_id"] for x in target_jobs]
for jid in args.arguments:
job = client.get_job(jid)
for attachment in job['attachments']:
print('downloading %s' % attachment['name'])
with open(attachment['name'], 'wb') as f:
f.write(client.download_job_attachment(id, attachment['name']))
elif args.command == 'execute':
if target_jobs:
print('Execute command does not support job filtering')
program_result.return_code = -1
return(program_result)
inputs = [{'name': basename(a), 'filename': a} for a in args.arguments]
if args.verbose:
for i in inputs:
print("Uploading %s as attachment name %s" % (i['filename'], i['name']))
execute_job(client, inputs, args.verbose, args.details, args.nodelete)
else:
print("Unknown command: %s" % args.command)
program_result.return_code = -1
return(program_result)
return(program_result)
def get(self):
global communities
global policeDistricts
global distances
global deployments
global crimecounts
global totalCoverage
global mapCoverage
global mapCrimes
global mapDeploys
global distanceCost
global fairness
global n_crimes_per_patrol
global crimetype_weights
if (deployments is None) or (communities is None) or (
policeDistricts is None) or (distances is None) or not loaded:
return {
'message': 'No deployment plan loaded.',
'result': 'failed'
}
# Get the passed arguments
argparser = reqparse.RequestParser()
argparser.add_argument('useFairness')
argparser.add_argument('minOnePatrolPerComm')
args = argparser.parse_args()
if json.loads(args['useFairness']) == 'yes':
useFairness = True
else:
useFairness = False
if json.loads(args['minOnePatrolPerComm']) == 'yes':
minOnePatrolPerComm = True
else:
minOnePatrolPerComm = False
print("Building model...")
# Create model
context = Context.make_default_context()
model = Model(name="CrimeDeployment", context=context)
print("Creating variables...")
# Create model variables
# Assignment of patrols from each district to each community is a separate variable
deployments_cpx = {}
deployed_cpx = defaultdict(int)
ethnicity_count = {0: 0, 1: 0}
for community in communities:
comm_id = communities[community]['id']
deployments_cpx[comm_id] = {'total': 0}
if (communities[community]['ethnicity']
== 0) or (communities[community]['ethnicity'] == 1):
ethnicity_count[1] += 1
else:
ethnicity_count[0] += 1
for district in policeDistricts:
dist_id = policeDistricts[district]['id']
max_patrols = policeDistricts[district]['total_patrols']
deployments_cpx[comm_id][dist_id] = model.integer_var(
lb=0,
ub=max_patrols,
name="c" + str(comm_id) + "d" + str(dist_id))
deployed_cpx[dist_id] += deployments_cpx[comm_id][dist_id]
deployments_cpx[comm_id]['total'] += deployments_cpx[comm_id][
dist_id]
print("Creating constraints...")
# Add model constraints
# Sum of deployed units over all communities can't be higher than number of available patrols
for district in policeDistricts:
dist_id = policeDistricts[district]['id']
max_patrols = policeDistricts[district]['total_patrols']
model.add_constraint(deployed_cpx[dist_id] <= max_patrols)
model.add_constraint(deployed_cpx[dist_id] >= 0)
# At least one patrol per community
if minOnePatrolPerComm:
for community in communities:
comm_id = communities[community]['id']
model.add_constraint(deployments_cpx[comm_id]['total'] >= 1)
print("Calculating objective...")
print(" - Crime Coverage")
# Calculate crime coverage in this deployment
# We check the coverage, but also apply a penalty for coverages larger than a set threshold
coverages = []
totalDeployed = []
totalCrimes = []
penalties = []
ethnicityDeployed = {0: [], 1: []}
for community in communities:
if crimecounts[communities[community]
['code']]['weighted_count'] != 0:
coverage = ((deployments_cpx[community]['total'] *
n_crimes_per_patrol) /
crimecounts[communities[community]
['code']]['weighted_count'])
coverages.append(coverage)
penalty = model.max(
0, coverage - (upper_penalty_threshold / 100)) + model.max(
0, (lower_penalty_threshold / 100) - coverage)
penalties.append(penalty)
totalDeployed.append(deployments_cpx[community]['total'] *
n_crimes_per_patrol)
totalCrimes.append(crimecounts[communities[community]['code']]
['weighted_count'])
if (communities[community]['ethnicity']
== 0) or (communities[community]['ethnicity'] == 1):
ethnicityDeployed[1].append(
deployments_cpx[community]['total'])
else:
ethnicityDeployed[0].append(
deployments_cpx[community]['total'])
avg_coverage = sum(coverages) / len(coverages)
avg_penalty = sum(penalties) / len(penalties)
citywideCoverage = sum(totalDeployed) / sum(totalCrimes)
citywidePenalty = model.max(
0, citywideCoverage - (upper_penalty_threshold / 100)) + model.max(
0, (lower_penalty_threshold / 100) - citywideCoverage)
fairness = model.abs((sum(ethnicityDeployed[0]) / ethnicity_count[0]) -
(sum(ethnicityDeployed[1]) / ethnicity_count[1]))
print(" - Distances")
# Calculate the distances units have to drive to fulfill deployments
distances_cpx = []
for comm in deployments_cpx:
for dist in deployments_cpx[comm]:
if (dist != 'total'):
distance = distances[(dist,
comm)] * deployments_cpx[comm][dist]
distances_cpx.append(distance)
distanceCost_cpx = sum(distances_cpx)
print(" - Combined Objective")
# Now build the objective function
obj = 1000000000 * citywidePenalty - 1000000000 * citywideCoverage + 1000000 * avg_penalty - 100000 * avg_coverage + distanceCost_cpx
if useFairness:
print(" - Add Fairness Statistic to Combined Objective")
# Add model objective
# First calculate the fairness of the deployment
obj = 10000 * fairness + obj
#model.add(maximize(obj))
model.minimize(obj)
print("Adding KPIs...")
# Add the KPIs we're interested in tracking
model.add_kpi(citywideCoverage, publish_name="City-wide Coverage")
model.add_kpi(citywidePenalty,
publish_name="City-wide Coverage Penalty")
model.add_kpi(avg_coverage, publish_name="Avg. Coverage")
model.add_kpi(avg_penalty, publish_name="Avg. Coverage Penalty")
model.add_kpi(distanceCost_cpx, publish_name="Total Distance")
if useFairness:
model.add_kpi(fairness,
publish_name="Fairness Simple Mean Difference")
model.add_kpi(obj, publish_name="Combined Objective")
print("Model ready:")
model.print_information()
print("Solving problem...")
# Solve the problem
msol = model.solve()
print("Done! Getting results...")
# Get result from solve
# if msol.solve_details['status']:
# return {'solve_status':msol.get_solve_status(),'message':'Optimization executed but aborted due to exceeding maximum run time.','result':'failed'}
# Get solution and place in deployments
model.report()
print(msol.solve_details)
# First reset any plan existing in memory and its KPIs
deployments = {}
mapCoverage = {}
mapDeploys = {}
totalCoverage = 0
distanceCost = 0
fairness = 0
for pd in db.session.query(PoliceDistrict):
policeDistricts[pd.id] = {
'id': pd.id,
'name': pd.name,
'total_patrols': pd.patrols,
'available_patrols': pd.patrols,
'deployed_patrols': 0
}
for community in communities:
comm_id = communities[community]['id']
deployments[comm_id] = defaultdict(int)
deployments[comm_id]['total'] = 0
for district in policeDistricts:
dist_id = policeDistricts[district]['id']
n_patrols = msol['c' + str(comm_id) + 'd' + str(dist_id)]
deployments[comm_id][dist_id] += n_patrols
deployments[comm_id]['total'] += n_patrols
policeDistricts[dist_id]['available_patrols'] -= n_patrols
policeDistricts[dist_id]['deployed_patrols'] += n_patrols
if crimecounts[communities[comm_id]
['code']]['weighted_count'] != 0:
mapCoverage[communities[comm_id]['code']] = ((deployments[comm_id]['total']*n_crimes_per_patrol)/ \
crimecounts[communities[comm_id]['code']]['weighted_count'])*100
else:
mapCoverage[communities[comm_id]['code']] = 0
mapDeploys[communities[comm_id]
['code']] = deployments[comm_id]['total']
distanceCost += n_patrols * distances[(dist_id, comm_id)]
# Calculate the total coverage of Crimes
totalcrimes = 0
totaldeploys = 0
for comm in deployments:
totalcrimes += crimecounts[communities[comm]
['code']]['weighted_count']
totaldeploys += deployments[comm]['total']
totalCoverage = (
(totaldeploys * n_crimes_per_patrol) / totalcrimes) * 100
# Calculate the deployment Fairness
# First calculate the t-statistic
t_stat, df = calculateFairnessTStat(communities, deployments)
# Now assign the p-value as the fairness
fairness = (1 - t.cdf(abs(t_stat), df)) * 2
# Convert to percentage value
fairness = fairness * 100
return {
'communities': communities,
'districts': policeDistricts,
'mapCoverage': mapCoverage,
'mapCrimes': mapCrimes,
'mapDeploys': mapDeploys,
'distanceCost': distanceCost,
'fairness': fairness,
'totalCoverage': totalCoverage,
'solve_status': msol.solve_details.status,
'message': 'Optimization executed succesfully.',
'result': 'success'
}
return mdl.objective_value
if __name__ == '__main__':
"""DOcloud credentials can be specified with url and api_key in the code block below.
Alternatively, Context.make_default_context() searches the PYTHONPATH for
the following files:
* cplex_config.py
* cplex_config_<hostname>.py
* docloud_config.py (must only contain context.solver.docloud configuration)
These files contain the credentials and other properties. For example,
something similar to::
context.solver.docloud.url = "https://docloud.service.com/job_manager/rest/v1"
context.solver.docloud.key = "example api_key"
"""
url = None
key = None
ctx = Context.make_default_context(url=url, key=key)
ctx.solver.docloud.print_information()
from docplex.mp.environment import Environment
env = Environment()
env.print_information()
solve_sports(context=ctx)
def ilp(graph, num_buses, size_bus, constraints):
edges = graph.edges()
buses = range(0, num_buses)
num_students = graph.nodes()
cxt = Context.make_default_context()
cxt.timeout = 1
with Model(name="solver", context=cxt) as mdl:
X_dict = {}
es = []
for s in graph.nodes():
name = "x" + str(s)
xslist = mdl.binary_var_list(keys=num_buses, name=name)
X_dict[s] = xslist
for s in graph.nodes():
cons = find_student_in_constraints(s, constraints)
xslist = X_dict[s]
constraining_values = []
for b in buses:
for C in cons:
relevant_dvs_names = ["x" + s + "_" + str(b) for s in C]
relevant_dvs = [
mdl.get_var_by_name(name)
for name in relevant_dvs_names
]
constraining_values.append(len(C) - mdl.sum(relevant_dvs))
rs = mdl.integer_var(name="r" + s)
expr = (rs <= mdl.min(constraining_values))
mdl.add_constraint(expr)
for e in edges:
name = "e" + e[0] + e[1]
edge_indic = mdl.binary_var_list(keys=num_buses, name=name)
es.extend(edge_indic)
obj = mdl.sum(es)
mdl.set_objective("max", obj)
for e in edges:
for b in buses:
u = e[0]
v = e[1]
name = "e" + u + v + "_" + str(b)
uexists = "x" + u + "_" + str(b)
vexists = "x" + v + "_" + str(b)
rsnameu = "r" + u
rsnamev = "r" + v
euvi = mdl.get_var_by_name(name)
xui = mdl.get_var_by_name(uexists)
xvi = mdl.get_var_by_name(vexists)
rsu = mdl.get_var_by_name(rsnameu)
rsv = mdl.get_var_by_name(rsnamev)
mdl.add_constraint((euvi <= xui))
mdl.add_constraint((euvi <= xvi))
mdl.add_constraint((euvi <= rsu))
mdl.add_constraint((euvi <= rsv))
for xslist in X_dict.values():
expr = (mdl.sum(xslist) == 1)
mdl.add_constraint(expr)
for b in buses:
relevant_list = [xslist[b] for xslist in X_dict.values()]
expr = (mdl.sum(relevant_list) <= size_bus)
expr2 = (mdl.sum(relevant_list) >= 1)
mdl.add_constraint(expr)
mdl.add_constraint(expr2)
mdl.set_time_limit(90)
solution = mdl.solve(log_output=True)
solved = []
for i in range(0, num_buses):
solved.append([])
iterv = solution.iter_var_values()
v = next(iterv)
i = 0
while i < (len(graph.nodes())):
name = str(v[0])
index = name.find('_')
student = name[1:index]
bus = int(name[index + 1:])
solved[bus].append(student)
i += 1
v = next(iterv)
print(solved)
assignment = utils.groups_to_assignment(solved)
if not (utils.is_valid(graph, num_buses, size_bus, assignment)):
print('\n')
print('NEET')
print('\n')
return None
print('yeet')
return assignment
to_arcs = []
from_arcs = []
for i in Arcs_modified2:
if (i[1] not in to_arcs):
to_arcs.append(i[1])
if (i[0] not in from_arcs):
from_arcs.append(i[0])
#print(Arcs_modified2)
#print(to_arcs)
# using doCloud Trial version Validity till 18th december
url = "https://api-oaas.docloud.ibmcloud.com/job_manager/rest/v1"
key = "api_b14a53e6-482a-44b7-b1e2-83b44d232ba0"
ctx = Context.make_default_context(url=url, key=key)
# model Objective function and constraints
mdl3 = Model(name='min_distance3', context=ctx)
x = mdl3.binary_var_dict(Arcs_modified2, name='x')
#print(x)
mdl3.minimize(mdl3.sum(c[i, j] * x[i, j] for i, j in Arcs_modified2))
# origin constraint
mdl3.add_constraint(
mdl3.sum(x[i, j] for i, j in Arcs_modified2 if i == origin) == 1)
#destination constraint
mdl3.add_constraint(
mdl3.sum(x[i, j] for i, j in Arcs_modified2 if j == destination) == 1)
#flow balancing Eq
for i, j in Arcs_modified2:
the following files:
* cplex_config.py
* cplex_config_<hostname>.py
* docloud_config.py (must only contain context.solver.docloud configuration)
These files contain the credentials and other properties. For example,
something similar to::
context.solver.docloud.url = 'https://docloud.service.com/job_manager/rest/v1'
context.solver.docloud.key = 'example api_key'
'''
url = None
key = None
if url is None or key is None:
# create a default context and use credentials defined in there.
context = Context.make_default_context()
url = context.solver.docloud.url
key = context.solver.docloud.key
client = JobClient(url=url, api_key=key)
resp = client.execute(input=[
'diet_pandas.py', 'diet_food.csv', 'diet_nutrients.csv',
'diet_food_nutrients.csv'
],
output='solution.json',
load_solution=True,
log='logs.txt')