def customers_to_jsonschema(customers_data: SuperDict):
customersTimeWindows = [
SuperDict(id_customer=key, **tw) for key, time_windows in
customers_data.get_property("timewindows").items()
for tw in time_windows
]
orders = [
SuperDict(id_customer=key, **order)
for key, orders in customers_data.get_property("orders").items()
for order in orders
]
forecasts = [
SuperDict(id_customer=key, time=time, forecast=forecast)
for key, forecasts in customers_data.get_property("Forecast").items()
for time, forecast in enumerate(forecasts) if forecast > 0
]
allowedTrailers = [
SuperDict(id_location=key, id_trailer=id_trailer) for key, trailers in
customers_data.get_property("allowedTrailers").items()
for id_trailer in trailers
]
customers = drop_props_get_values(
customers_data,
["timewindows", "Forecast", "allowedTrailers", "orders"])
return SuperDict(
customers=customers,
customersTimeWindows=customersTimeWindows,
orders=orders,
forecasts=forecasts,
allowedTrailers=allowedTrailers,
)
def check_shift_16(self, shift, id_shift):
"""
returns {"exceeds": {(id_shift, id_location): quantity, ... } for
operation with quantity above trailer's capacity
"too_low": {(id_shift, id_location): quantity, ... } for
operation with quantity < min_quantity
}
"""
check = SuperDict(exceeds=SuperDict(), too_low=SuperDict())
for i in range(1, len(shift["route"]) + 1):
operation = shift["route"][i - 1]
location = operation["location"]
if not self.is_customer(location):
continue
call_in = self.instance.get_customer_property(location, "callIn")
if call_in == 0:
capacity = self.instance.get_customer_property(
location, "Capacity")
min_ope_quantity = self.instance.get_customer(location).get(
"MinOperationQuantity", 0)
if -operation["quantity"] > capacity:
check["exceeds"][(id_shift,
location)] = operation["quantity"]
elif -operation["quantity"] < min_ope_quantity:
check["too_low"][(id_shift,
location)] = operation["quantity"]
check = check.vfilter(lambda v: len(v) != 0)
return check
def read_excel(path: str, param_tables_names: list = None) -> dict:
"""
Read an entire excel file.
:param path: path of the excel file
:param param_tables_names: names of the parameter tables
:return: a dict with a list of dict (records format) for each table.
"""
is_xl_type(path)
try:
import openpyxl
except (ModuleNotFoundError, ImportError) as e:
raise Exception("You must install openpyxl package to use this method")
data = pd.read_excel(path, sheet_name=None)
data_tables = {
name: TupList(content.to_dict(orient="records")).vapply(
lambda v: SuperDict(v).vapply(lambda vv: format_value(vv)))
for name, content in data.items() if name not in param_tables_names
}
parameters_tables = {
t: SuperDict(read_param_table(path,
t)).vapply(lambda v: format_value(v))
for t in param_tables_names
}
return {**data_tables, **parameters_tables}
def sources_to_jsonschema(sources_data):
allowedTrailers = [
SuperDict(id_location=key, id_trailer=id_trailer) for key, trailers in
sources_data.get_property("allowedTrailers").items()
for id_trailer in trailers
]
sources = drop_props_get_values(sources_data, ["allowedTrailers"])
return SuperDict(sources=sources, allowedTrailers=allowedTrailers)
def check_shift_0607(self):
"""
returns {"negative": {(id_shift, id_location): quantity, ... } for operation resulting in negative trailer inventory
"above_capacity": {(id_shift, id_location): quantity, ... } for operation resulting in excessive trailer inventory
"final_inventory": {id_shift: 1, ...} for shifts with inconsistent final inventory
"initial_inventory": {id_shift: 1, ...} for shifts with inconsistent initial inventory }
"""
check = SuperDict(
negative=SuperDict(),
above_capacity=SuperDict(),
final_inventory=SuperDict(),
initial_inventory=SuperDict(),
)
last_trailer_quantity = [0] * len(self.instance.get_id_trailers())
aux_info_shift = self.get_aux_info_shift()
for trailer in self.instance.get_id_trailers():
last_trailer_quantity[
trailer] = self.instance.get_trailer_property(
trailer, "InitialQuantity")
for shift in sorted(list(self.solution.get_all_shifts()),
key=lambda x: x["departure_time"]):
driver = shift.get("driver", None)
trailer = shift.get("trailer", None)
id_shift = shift.get("id_shift", None)
if driver is None or trailer is None:
return check
# Constraints (06)
last_quantity = shift["initial_quantity"]
for operation in shift["route"]:
quantity = last_quantity + operation["quantity"]
location = operation["location"]
if round(quantity, 2) < 0:
check["negative"][(id_shift, location)] = quantity
elif round(quantity, 2) > self.instance.get_trailer_property(
trailer, "Capacity"):
check["above_capacity"][(id_shift, location)] = quantity
last_quantity = quantity
# Constraints (07)
if round(aux_info_shift[id_shift]["final_inventory"], 2) != round(
last_quantity, 2):
check["final_inventory"][id_shift] = 1
if round(last_trailer_quantity[trailer], 2) != round(
shift["initial_quantity"], 2):
check["initial_inventory"][id_shift] = 1
last_trailer_quantity[trailer] = last_quantity
check = check.vfilter(lambda v: len(v) != 0)
return check
def matrix_to_dict(matrix, key, func):
matrix = TupList(matrix).vapply(lambda v: v[key])
result = SuperDict()
for L1, row in enumerate(matrix):
for L2, col in enumerate(row):
result[L1, L2] = func(col)
return result
def index_list(table, index, _list):
"""
indexes the table by index and returns a dictionary with _list keys
"""
return (TupList(table).to_dict(result_col=_list, indices=[
index
]).vapply(lambda v: v.vapply(lambda vv: SuperDict(zip(_list, vv)))))
def from_dict(cls, data_dict):
# main tables
trailers = _index(data_dict["trailers"])
drivers = get_drivers(data_dict)
customers = get_customers(data_dict)
sources = get_sources(data_dict)
# dist and time matrices
matrices = {(el["L1"], el["L2"]): el for el in data_dict["matrices"]}
data_out = SuperDict(
trailers=trailers,
drivers=drivers,
customers=customers,
sources=sources,
matrices=matrices,
coordinates=data_dict.get("coordinates", [])
)
# other parameters
for param in ["unit", "horizon"]:
data_out[param] = data_dict["parameters"][param]
data_out["bases"] = data_dict["bases"][0]
return cls(data_out)
def get_aux_info_shift(self):
"""
Returns a dictionary containing information about each shift of self.solution,
like : the ending time of the shift (in minutes), the inventory of the trailer at the end of the shift,
the total duration of the shift and its driving duration
For example: {2: {'arrival_time': 367, 'final_inventory': 1760, 'duration': 187, 'driving_duration': 160},
3: {'arrival_time': 512, 'final_inventory': 950, 'duration': 203, 'driving_duration': 180} }
"""
shifts = self.solution.get_shifts_dict()
result = SuperDict()
for id_shift, shift in shifts.items():
cumulated_driving_time = 0
for s, (stop, next_stop) in enumerate(
zip(shift['route'], shift['route'][1:])):
stop["cumulated_driving_time"] = cumulated_driving_time
cumulated_driving_time += self.instance.get_time_between(
stop["location"],
next_stop["location"],
)
shift["route"][-1][
"cumulated_driving_time"] = cumulated_driving_time
result[id_shift] = dict(
arrival_time=shift["route"][-1]["arrival"],
final_inventory=shift["initial_quantity"] +
sum([step["quantity"] for step in shift["route"]]),
duration=shift["route"][-1]["arrival"] -
shift["departure_time"],
driving_duration=sum(
self.instance.get_time_between(
stop["location"], next_stop["location"]) for stop,
next_stop in zip(shift["route"], shift["route"][1:])))
return result
def solve(self, options: dict) -> dict:
model = pl.LpProblem("rostering", pl.LpMaximize)
# Variables:
self.create_variables()
# Constraints:
model = self.create_constraints(model)
# print(model)
# Solver and solve
mat_solver = pl.PULP_CBC_CMD(
gapRel=0.001,
timeLimit=options.get("timeLimit", 240),
msg=options.get("msg", False),
)
status = model.solve(mat_solver)
# Check status
if model.sol_status not in [pl.LpSolutionIntegerFeasible, pl.LpSolutionOptimal]:
return dict(status=status, status_sol=SOLUTION_STATUS_INFEASIBLE)
work_assignments = (
self.works.vfilter(lambda v: pl.value(v))
.keys_tl()
.vapply(lambda v: dict(id_employee=v[1], time_slot=v[0]))
)
self.solution = Solution.from_dict(SuperDict(works=work_assignments))
return dict(status=status, status_sol=SOLUTION_STATUS_FEASIBLE)
def check_solution(self):
return SuperDict({
**self.check_shifts(),
**self.check_sites(),
**self.check_resources(),
**self.check_service_quality(),
}).vfilter(lambda v: len(v))
def check_qs_02(self):
"""
returns {location: nb_run_outs, ... } for locations experiencing runouts
"""
check = SuperDict()
nb_run_outs = 0
site_inventories = self.calculate_inventories()
for site in site_inventories.keys():
site_inventory = site_inventories[site]
nb_inventory_run_out = 0
if not site_inventories[site]:
continue
if site > self.nb_customers + self.nb_sources + 1:
continue
if self.is_customer(site):
customer = self.instance.get_customer(site)
nb_inventory_run_out += sum(
1 for i in range(self.horizon)
if round(site_inventory["tank_quantity"][i], 2) < 0
if customer["callIn"] == 0)
nb_run_outs += nb_inventory_run_out
if nb_inventory_run_out != 0:
check[site] = nb_inventory_run_out
return check
def inherit(self):
all_classes = set(self.parents.keys())
not_treated = set(all_classes)
treated = {"db.Model"}
while not_treated:
for model in not_treated:
parent = self.parents[model]
if parent is None:
treated.add(model)
continue
if parent not in treated:
continue
treated.add(model)
if parent == "db.Model":
continue
table_name = self.model_table[model]
parent_props = self.data[
self.model_table[parent]]["items"]["properties"]
parent_requirements = self.data[
self.model_table[parent]]["items"]["required"]
self.data[table_name]["items"]["properties"] = SuperDict(
**parent_props,
**self.data[table_name]["items"]["properties"])
self.data[table_name]["items"][
"required"] += parent_requirements
not_treated -= treated
if not self.leave_bases:
self.data = self.data.vfilter(lambda v: not v.get("remove", False))
def sub_check_qs_03(self, shift, id_shift):
"""
returns {(shift_id, location): 1, ... } for deliveries not related to an order to callIn customer
"""
check = SuperDict()
for operation in shift["route"][:-1]:
location = operation["location"]
if not self.is_valid_location(location):
continue
if not self.is_customer(location):
continue
customer = self.instance.get_customer(location)
if customer["callIn"] == 0 or not customer["orders"]:
continue
tw_found = False
ind_tw = 0
while ind_tw < len(customer["orders"]) and not tw_found:
order = customer["order"][ind_tw]
if order["earliestTime"] <= operation["arrival"] <= order[
"latestTime"]:
tw_found = True
ind_tw += 1
if not tw_found:
check[(id_shift, location)] = 1
return check
def drivers_to_jsonschema(drivers_data: SuperDict):
driversTimeWindows = [
dict(id_driver=key, **tw) for key, time_windows in
drivers_data.get_property("timewindows").items() for tw in time_windows
]
driversTrailers = [
SuperDict(id_driver=key, id_trailer=trailer)
for key, trailers in drivers_data.get_property("trailer").items()
for trailer in trailers
]
drivers = drop_props_get_values(drivers_data, ["trailer", "timewindows"])
return SuperDict(
drivers=drivers,
driversTrailers=driversTrailers,
driversTimeWindows=driversTimeWindows,
)
def __init__(self, data: dict):
super().__init__(data)
# Stores a list of the starting date of each week ordered
self.weeks = TupList()
# First object stores a list of the dates ordered,
# the second the properties for each date.
self.dates = TupList()
self.dates_properties = SuperDict()
# First object stores a list of the time slots ordered,
# the second the properties for each one.
self.time_slots = TupList()
self.time_slots_properties = SuperDict()
self.cache_properties()
def from_dict(cls, data: dict) -> "Solution":
data_p = {
el: {(v["id_employee"], v["time_slot"]): v
for v in data[el]}
for el in ["works"]
}
return cls(SuperDict(data_p))
def solve_one_iteration(self, solver, used_routes, previous_value, current_round):
if 0 < current_round <= self.limit_artificial_round + 1:
self.generate_new_routes()
self.artificial_quantities = dict()
old_used_routes = pickle.loads(pickle.dumps(used_routes, -1))
previous_routes_infos = [
(shift["id_shift"], shift["trailer"], shift["driver"])
for shift in self.solution.get_all_shifts()
]
if current_round > 0:
selected_routes = self.select_routes(self.nb_routes)
used_routes = SuperDict({**used_routes, **selected_routes})
self.initialize_parameters(used_routes)
model = self.new_model(
used_routes,
previous_routes_infos,
current_round <= self.limit_artificial_round,
)
status = model.solve(solver=solver)
if status == 1:
self.to_solution(model, used_routes, current_round)
if current_round > self.limit_artificial_round:
self.check_and_save(current_round)
if status != 1 or current_round == 0:
used_routes = old_used_routes
return used_routes, None
if not (
previous_value is None
or pl.value(model.objective) < previous_value
or (
(current_round > self.limit_artificial_round)
and (self.last_solution_round <= self.limit_artificial_round)
)
):
return old_used_routes, previous_value
keep = (
self.route_var.vfilter(lambda v: pl.value(v) > 0.5)
.keys_tl()
.take(0)
.to_dict(None)
.vapply(lambda v: True)
)
used_routes = {
r: route for r, route in used_routes.items() if keep.get(r, False)
}
if current_round > self.limit_artificial_round:
previous_value = pl.value(model.objective)
self.last_solution_round = current_round
return used_routes, previous_value
def check_shift_03(self, shift, id_shift):
"""
returns {"wrong_index": {(id_shift, id_location): 1, ... } for non existent locations,
"setup_time": {(id_shift, id_location): 1, ... } for operation that does not respect setup time }
"""
check = SuperDict(wrong_index=SuperDict(), setup_time=SuperDict())
id_operation = 1
for operation in shift["route"][:-1]:
if not self.is_valid_location(operation["location"]):
check["wrong_index"][(id_shift, id_operation)] = 1
continue
location = operation["location"]
if location > 0:
setup_time = self.instance.get_location_property(
location, "setupTime")
if operation["departure"] < operation["arrival"] + setup_time:
check["setup_time"][(id_shift, location)] = 1
id_operation += 1
return check
def check_solution(self, *args, **kwargs) -> dict:
return SuperDict(
availability=self.check_availability(),
restricted_flows=self.check_restricted_flows(),
demand=self.check_demand(),
second_doses=self.check_second_dose(),
warehouse_capacities=self.check_warehouse_capacity(),
consistency_warehouses=self.check_consistency_warehouses(),
consistency_suppliers=self.check_consistency_suppliers(),
)
def from_dict(cls, data_dict):
rows = SuperDict(
{
(el["id_shift"], el["position"]): SuperDict(el)
for el in data_dict["details_shifts"]
}
)
details = rows.to_dictdict().vapply(
lambda v: v.values_tl().sorted(key=lambda x: x["position"])
)
data = {
shift["id_shift"]: SuperDict(shift) for shift in data_dict["info_shifts"]
}
for shift in data:
data[shift]["route"] = details[shift]
return cls(data)
def get_demand(self) -> SuperDict:
"""
Returns a SuperDict indexed by the time slot (string) and the demand as value
For example: {"2021-09-06T07:00": 10, "2021-09-06T08:00": 15, ...}
"""
return SuperDict(
{
self._get_time_slot_string(ts): self._filter_demand(ts)
for ts in self.time_slots
}
)
def solve(self, options: dict):
distance = (
self.instance.get_arcs()
.to_dict(result_col=["w"], indices=["n1", "n2"], is_list=False)
.kfilter(lambda k: k[0] != k[1])
)
model = cp_model.CpModel()
create_literal = lambda i, j: model.NewBoolVar("%i follows %i" % (j, i))
literals = distance.kapply(lambda k: create_literal(*k))
arcs = literals.to_tuplist()
model.AddCircuit(arcs)
model.Minimize(sum((literals * distance).values()))
solver = cp_model.CpSolver()
if options.get("msg", False):
solver.parameters.log_search_progress = True
# To benefit from the linearization of the circuit constraint.
solver.parameters.linearization_level = 2
solver.parameters.max_time_in_seconds = options.get("timeLimit", 10)
if "threads" in options:
solver.parameters.num_search_workers = options["threads"]
status = solver.Solve(model)
if options.get("msg", False):
print(solver.ResponseStats())
status_conv = {
cp_model.OPTIMAL: STATUS_OPTIMAL,
cp_model.INFEASIBLE: STATUS_INFEASIBLE,
cp_model.UNKNOWN: STATUS_UNDEFINED,
cp_model.MODEL_INVALID: STATUS_UNDEFINED,
}
if status not in [cp_model.OPTIMAL, cp_model.FEASIBLE]:
return dict(
status=status_conv.get(status), status_sol=SOLUTION_STATUS_INFEASIBLE
)
next = (
literals.vapply(solver.BooleanValue)
.vfilter(lambda v: v)
.keys_tl()
.to_dict(1, is_list=False)
)
first = next.keys_tl(0)
current_node = first
solution = TupList([first])
while True:
current_node = next[current_node]
if current_node == first:
break
solution.append(current_node)
nodes = solution.kvapply(lambda k, v: SuperDict(pos=k, node=v))
self.solution = Solution(dict(route=nodes))
return dict(status=status_conv.get(status), status_sol=SOLUTION_STATUS_FEASIBLE)
def check_shift_05(self, shift, id_shift, trailer):
"""
returns {(shift_id, location): 1, ... } for incompatible shift-location combination
"""
check = SuperDict()
for operation in shift["route"][:-1]:
location = operation["location"]
if self.is_customer_or_source(location):
if trailer not in self.instance.get_location_property(
location, "allowedTrailers"):
check[(id_shift, location)] = 1
return check
def create_variables(self):
self.works = pl.LpVariable.dicts(
"works",
self.employee_ts_availability,
lowBound=0,
upBound=1,
cat=pl.LpBinary,
)
self.works = SuperDict(self.works)
self.starts = pl.LpVariable.dicts(
"starts",
self.employee_ts_availability,
lowBound=0,
upBound=1,
cat=pl.LpBinary,
)
self.starts = SuperDict(self.starts)
def __init__(self, instance, solution=None):
super().__init__(instance, solution)
self.log = ""
self.solver = "MIP Model"
self.routes_generator = RoutesGenerator(self.instance)
self.range_hours = list(range(self.horizon))
self.routes = dict()
self.unused_routes = dict()
self.value_greedy = None
self.interval_routes = 6
self.nb_routes = 1000
self.start_time = datetime.now()
self.start_time_string = datetime.now().strftime("%d.%m-%Hh%M")
self.print_log = False
self.save_results = False
self.artificial_quantities = dict()
self.limit_artificial_round = 0
self.last_solution_round = -1
self.solution_greedy = None
self.locations_in = dict()
self.unique_locations_in = dict()
self.hour_of_visit = dict()
self.k_visit_hour = dict()
self.nb_visits = dict()
self.coef_inventory_conservation = 0.8
self.time_limit = 100000
# Variables
self.route_var = SuperDict()
self.artificial_quantities_var = SuperDict()
self.artificial_binary_var = SuperDict()
self.inventory_var = SuperDict()
self.quantity_var = SuperDict()
self.trailer_quantity_var = SuperDict()
def check_shift_11(self, shift, id_shift):
"""
returns {(shift_id, location): quantity, ... } for operation with wrong sign
"""
check = SuperDict()
for i in range(1, len(shift["route"]) + 1):
operation = shift["route"][i - 1]
location = operation["location"]
if self.is_source(location) and operation["quantity"] < 0:
check[(id_shift, location)] = operation["quantity"]
elif self.is_customer(location) and operation["quantity"] > 0:
check[(id_shift, location)] = operation["quantity"]
return check
def check_sites(self):
"""
returns {"site_inventory_negative": {(location, time): inventory, ... } for inventories > tank_capacity
"site_inventory_exceeds": {(location, time): inventory, ... } for negative inventories
"site_doesntexist": {location: 1} for nonexistent locations
}
"""
check = SuperDict(
site_inventory_negative=SuperDict(),
site_inventory_exceeds=SuperDict(),
site_doesntexist=SuperDict(),
)
operation_quantities = []
for location in range(1 + self.nb_sources + self.nb_customers):
operation_quantities.append([0] * self.horizon)
site_inventories = self.calculate_inventories()
for site_inventory in site_inventories.values():
location = site_inventory["location"]
if not self.is_valid_location(location):
check["site_doesntexist"][location] = 1
continue
if location <= 1 + self.nb_sources:
continue
call_in = self.instance.get_customer_property(location, "callIn")
if call_in == 1:
continue
for i in range(self.horizon):
if round(site_inventory["tank_quantity"][i], 3) < 0:
check["site_inventory_negative"][
location, i] = site_inventory["tank_quantity"][i]
if round(site_inventory["tank_quantity"][i],
3) > self.instance.get_customer_property(
location, "Capacity"):
check["site_inventory_exceeds"][
location, i] = site_inventory["tank_quantity"][i]
check = check.vfilter(lambda v: len(v) != 0)
return check
def _get_dates_properties(self) -> SuperDict:
"""
Returns a SuperDict with dates as key and its dict properties as a value
For example: {datetime(2021, 9, 6, 0, 0, 0): {"string": "2021-09-06", "week": 36}, ...}
"""
return SuperDict(
{
date: {
"string": get_date_string_from_ts(date),
"week": get_week_from_ts(date),
}
for date in self.dates
}
)
def calculate_inventories(self):
"""
Calculates the inventory of each customer at each hour of the time horizon
:return: A dictionary whose keys are the indexes of the customers
and whose values are dictionaries containing two elements:
- a list 'tank_quantity' containing the value of the inventory at each hour
- an integer 'location' corresponding to the index of the customer
For example: {2: {'tank_inventory': [15000, 14000, 13000, 16000, ... , 14000, 13000], 'location': 2},
3: {'tank_inventory': [4000, 4000, 4000, 1000, ..., 3000, 3000], 'location': 3}}
"""
customers = self.instance.get_id_customers()
_get_customer = lambda c, p: self.instance.get_location_property(c, p)
# we need three things: consumption, initial stock and arrivals
# we store each as a dictionary where each customer has a numpy array of length = horizon
# 1. we get consumptions from forecasts
consumption = (customers.to_dict(None).vapply(
_get_customer, "Forecast").vapply(lambda v: -np.array(v)).vapply(
lambda v: v[0:self.horizon]))
# 2. we get initial tanks and assign them to the first period
initial_tank = consumption.vapply(lambda v: np.zeros(self.horizon))
for k, v in initial_tank.items():
v[0] = _get_customer(k, "InitialTankQuantity")
# 3. we now use the solution to get the arrivals with routes
shifts = self.solution.get_all_shifts()
all_operations = TupList(operation
for route_list in shifts.take("route")
for operation in route_list)
# for each route we take the location, the time and how much.
# we only get customers and round the time and make positive the quantity
arrivals_tup = (all_operations.take(
["location", "arrival",
"quantity"]).vfilter(lambda v: self.is_customer(v[0])).vapply(
lambda v: (v[0], floor(v[1] / self.unit), -round(v[2], 3))))
# we initialize at 0 and increase when a truck arrives:
arrivals = consumption.vapply(lambda v: np.zeros(self.horizon))
for customer, time, quantity in arrivals_tup:
arrivals[customer][time] += quantity
# we take advantage of both pytups broadcasting and numpys broadcasting
# then we accumulate over periods
stocks = (consumption + arrivals + initial_tank).vapply(np.cumsum)
site_inventories = SuperDict()
for _id, quantity_arr in stocks.items():
site_inventories[_id] = dict(tank_quantity=quantity_arr,
location=_id)
return site_inventories