def check_shift_16(self, shift, id_shift):
"""
returns {"exceeds": {(id_shift, id_location): quantity, ... } for
operation with quantity above trailer's capacity
"too_low": {(id_shift, id_location): quantity, ... } for
operation with quantity < min_quantity
}
"""
check = SuperDict(exceeds=SuperDict(), too_low=SuperDict())
for i in range(1, len(shift["route"]) + 1):
operation = shift["route"][i - 1]
location = operation["location"]
if not self.is_customer(location):
continue
call_in = self.instance.get_customer_property(location, "callIn")
if call_in == 0:
capacity = self.instance.get_customer_property(
location, "Capacity")
min_ope_quantity = self.instance.get_customer(location).get(
"MinOperationQuantity", 0)
if -operation["quantity"] > capacity:
check["exceeds"][(id_shift,
location)] = operation["quantity"]
elif -operation["quantity"] < min_ope_quantity:
check["too_low"][(id_shift,
location)] = operation["quantity"]
check = check.vfilter(lambda v: len(v) != 0)
return check
def read_excel(path: str, param_tables_names: list = None) -> dict:
"""
Read an entire excel file.
:param path: path of the excel file
:param param_tables_names: names of the parameter tables
:return: a dict with a list of dict (records format) for each table.
"""
is_xl_type(path)
try:
import openpyxl
except (ModuleNotFoundError, ImportError) as e:
raise Exception("You must install openpyxl package to use this method")
data = pd.read_excel(path, sheet_name=None)
data_tables = {
name: TupList(content.to_dict(orient="records")).vapply(
lambda v: SuperDict(v).vapply(lambda vv: format_value(vv)))
for name, content in data.items() if name not in param_tables_names
}
parameters_tables = {
t: SuperDict(read_param_table(path,
t)).vapply(lambda v: format_value(v))
for t in param_tables_names
}
return {**data_tables, **parameters_tables}
def solve_one_iteration(self, solver, used_routes, previous_value, current_round):
if 0 < current_round <= self.limit_artificial_round + 1:
self.generate_new_routes()
self.artificial_quantities = dict()
old_used_routes = pickle.loads(pickle.dumps(used_routes, -1))
previous_routes_infos = [
(shift["id_shift"], shift["trailer"], shift["driver"])
for shift in self.solution.get_all_shifts()
]
if current_round > 0:
selected_routes = self.select_routes(self.nb_routes)
used_routes = SuperDict({**used_routes, **selected_routes})
self.initialize_parameters(used_routes)
model = self.new_model(
used_routes,
previous_routes_infos,
current_round <= self.limit_artificial_round,
)
status = model.solve(solver=solver)
if status == 1:
self.to_solution(model, used_routes, current_round)
if current_round > self.limit_artificial_round:
self.check_and_save(current_round)
if status != 1 or current_round == 0:
used_routes = old_used_routes
return used_routes, None
if not (
previous_value is None
or pl.value(model.objective) < previous_value
or (
(current_round > self.limit_artificial_round)
and (self.last_solution_round <= self.limit_artificial_round)
)
):
return old_used_routes, previous_value
keep = (
self.route_var.vfilter(lambda v: pl.value(v) > 0.5)
.keys_tl()
.take(0)
.to_dict(None)
.vapply(lambda v: True)
)
used_routes = {
r: route for r, route in used_routes.items() if keep.get(r, False)
}
if current_round > self.limit_artificial_round:
previous_value = pl.value(model.objective)
self.last_solution_round = current_round
return used_routes, previous_value
def sources_to_jsonschema(sources_data):
allowedTrailers = [
SuperDict(id_location=key, id_trailer=id_trailer) for key, trailers in
sources_data.get_property("allowedTrailers").items()
for id_trailer in trailers
]
sources = drop_props_get_values(sources_data, ["allowedTrailers"])
return SuperDict(sources=sources, allowedTrailers=allowedTrailers)
def customers_to_jsonschema(customers_data: SuperDict):
customersTimeWindows = [
SuperDict(id_customer=key, **tw) for key, time_windows in
customers_data.get_property("timewindows").items()
for tw in time_windows
]
orders = [
SuperDict(id_customer=key, **order)
for key, orders in customers_data.get_property("orders").items()
for order in orders
]
forecasts = [
SuperDict(id_customer=key, time=time, forecast=forecast)
for key, forecasts in customers_data.get_property("Forecast").items()
for time, forecast in enumerate(forecasts) if forecast > 0
]
allowedTrailers = [
SuperDict(id_location=key, id_trailer=id_trailer) for key, trailers in
customers_data.get_property("allowedTrailers").items()
for id_trailer in trailers
]
customers = drop_props_get_values(
customers_data,
["timewindows", "Forecast", "allowedTrailers", "orders"])
return SuperDict(
customers=customers,
customersTimeWindows=customersTimeWindows,
orders=orders,
forecasts=forecasts,
allowedTrailers=allowedTrailers,
)
def __init__(self,
path,
output_path=None,
ignore_files=None,
leave_bases=False):
self.path = path
self.tmp_path = os.path.join(os.getcwd(), "tmp_files")
self.output_path = output_path or "./output_schema.json"
self.ignore_files = ignore_files or []
self.leave_bases = leave_bases or False
self.parents = dict()
self.data = SuperDict()
self.model_table = dict()
self.table_model = dict()
def matrix_to_dict(matrix, key, func):
matrix = TupList(matrix).vapply(lambda v: v[key])
result = SuperDict()
for L1, row in enumerate(matrix):
for L2, col in enumerate(row):
result[L1, L2] = func(col)
return result
def index_list(table, index, _list):
"""
indexes the table by index and returns a dictionary with _list keys
"""
return (TupList(table).to_dict(result_col=_list, indices=[
index
]).vapply(lambda v: v.vapply(lambda vv: SuperDict(zip(_list, vv)))))
def from_dict(cls, data_dict):
# main tables
trailers = _index(data_dict["trailers"])
drivers = get_drivers(data_dict)
customers = get_customers(data_dict)
sources = get_sources(data_dict)
# dist and time matrices
matrices = {(el["L1"], el["L2"]): el for el in data_dict["matrices"]}
data_out = SuperDict(
trailers=trailers,
drivers=drivers,
customers=customers,
sources=sources,
matrices=matrices,
coordinates=data_dict.get("coordinates", [])
)
# other parameters
for param in ["unit", "horizon"]:
data_out[param] = data_dict["parameters"][param]
data_out["bases"] = data_dict["bases"][0]
return cls(data_out)
def inherit(self):
all_classes = set(self.parents.keys())
not_treated = set(all_classes)
treated = {"db.Model"}
while not_treated:
for model in not_treated:
parent = self.parents[model]
if parent is None:
treated.add(model)
continue
if parent not in treated:
continue
treated.add(model)
if parent == "db.Model":
continue
table_name = self.model_table[model]
parent_props = self.data[
self.model_table[parent]]["items"]["properties"]
parent_requirements = self.data[
self.model_table[parent]]["items"]["required"]
self.data[table_name]["items"]["properties"] = SuperDict(
**parent_props,
**self.data[table_name]["items"]["properties"])
self.data[table_name]["items"][
"required"] += parent_requirements
not_treated -= treated
if not self.leave_bases:
self.data = self.data.vfilter(lambda v: not v.get("remove", False))
def check_solution(self):
return SuperDict({
**self.check_shifts(),
**self.check_sites(),
**self.check_resources(),
**self.check_service_quality(),
}).vfilter(lambda v: len(v))
def solve(self, options: dict) -> dict:
model = pl.LpProblem("rostering", pl.LpMaximize)
# Variables:
self.create_variables()
# Constraints:
model = self.create_constraints(model)
# print(model)
# Solver and solve
mat_solver = pl.PULP_CBC_CMD(
gapRel=0.001,
timeLimit=options.get("timeLimit", 240),
msg=options.get("msg", False),
)
status = model.solve(mat_solver)
# Check status
if model.sol_status not in [pl.LpSolutionIntegerFeasible, pl.LpSolutionOptimal]:
return dict(status=status, status_sol=SOLUTION_STATUS_INFEASIBLE)
work_assignments = (
self.works.vfilter(lambda v: pl.value(v))
.keys_tl()
.vapply(lambda v: dict(id_employee=v[1], time_slot=v[0]))
)
self.solution = Solution.from_dict(SuperDict(works=work_assignments))
return dict(status=status, status_sol=SOLUTION_STATUS_FEASIBLE)
def check_qs_02(self):
"""
returns {location: nb_run_outs, ... } for locations experiencing runouts
"""
check = SuperDict()
nb_run_outs = 0
site_inventories = self.calculate_inventories()
for site in site_inventories.keys():
site_inventory = site_inventories[site]
nb_inventory_run_out = 0
if not site_inventories[site]:
continue
if site > self.nb_customers + self.nb_sources + 1:
continue
if self.is_customer(site):
customer = self.instance.get_customer(site)
nb_inventory_run_out += sum(
1 for i in range(self.horizon)
if round(site_inventory["tank_quantity"][i], 2) < 0
if customer["callIn"] == 0)
nb_run_outs += nb_inventory_run_out
if nb_inventory_run_out != 0:
check[site] = nb_inventory_run_out
return check
def get_aux_info_shift(self):
"""
Returns a dictionary containing information about each shift of self.solution,
like : the ending time of the shift (in minutes), the inventory of the trailer at the end of the shift,
the total duration of the shift and its driving duration
For example: {2: {'arrival_time': 367, 'final_inventory': 1760, 'duration': 187, 'driving_duration': 160},
3: {'arrival_time': 512, 'final_inventory': 950, 'duration': 203, 'driving_duration': 180} }
"""
shifts = self.solution.get_shifts_dict()
result = SuperDict()
for id_shift, shift in shifts.items():
cumulated_driving_time = 0
for s, (stop, next_stop) in enumerate(
zip(shift['route'], shift['route'][1:])):
stop["cumulated_driving_time"] = cumulated_driving_time
cumulated_driving_time += self.instance.get_time_between(
stop["location"],
next_stop["location"],
)
shift["route"][-1][
"cumulated_driving_time"] = cumulated_driving_time
result[id_shift] = dict(
arrival_time=shift["route"][-1]["arrival"],
final_inventory=shift["initial_quantity"] +
sum([step["quantity"] for step in shift["route"]]),
duration=shift["route"][-1]["arrival"] -
shift["departure_time"],
driving_duration=sum(
self.instance.get_time_between(
stop["location"], next_stop["location"]) for stop,
next_stop in zip(shift["route"], shift["route"][1:])))
return result
def sub_check_qs_03(self, shift, id_shift):
"""
returns {(shift_id, location): 1, ... } for deliveries not related to an order to callIn customer
"""
check = SuperDict()
for operation in shift["route"][:-1]:
location = operation["location"]
if not self.is_valid_location(location):
continue
if not self.is_customer(location):
continue
customer = self.instance.get_customer(location)
if customer["callIn"] == 0 or not customer["orders"]:
continue
tw_found = False
ind_tw = 0
while ind_tw < len(customer["orders"]) and not tw_found:
order = customer["order"][ind_tw]
if order["earliestTime"] <= operation["arrival"] <= order[
"latestTime"]:
tw_found = True
ind_tw += 1
if not tw_found:
check[(id_shift, location)] = 1
return check
def drivers_to_jsonschema(drivers_data: SuperDict):
driversTimeWindows = [
dict(id_driver=key, **tw) for key, time_windows in
drivers_data.get_property("timewindows").items() for tw in time_windows
]
driversTrailers = [
SuperDict(id_driver=key, id_trailer=trailer)
for key, trailers in drivers_data.get_property("trailer").items()
for trailer in trailers
]
drivers = drop_props_get_values(drivers_data, ["trailer", "timewindows"])
return SuperDict(
drivers=drivers,
driversTrailers=driversTrailers,
driversTimeWindows=driversTimeWindows,
)
def from_dict(cls, data: dict) -> "Solution":
data_p = {
el: {(v["id_employee"], v["time_slot"]): v
for v in data[el]}
for el in ["works"]
}
return cls(SuperDict(data_p))
def __init__(self, data: dict):
super().__init__(data)
# Stores a list of the starting date of each week ordered
self.weeks = TupList()
# First object stores a list of the dates ordered,
# the second the properties for each date.
self.dates = TupList()
self.dates_properties = SuperDict()
# First object stores a list of the time slots ordered,
# the second the properties for each one.
self.time_slots = TupList()
self.time_slots_properties = SuperDict()
self.cache_properties()
def check_availability(self):
availability_dict = self.instance.get_availability()
purchases_dict = self.solution.get_amount_supplied().kfilter(
lambda k: k[0] in self.instance.get_suppliers()
)
check_dict = SuperDict.from_dict(
{k: availability_dict[k] - purchases_dict[k] for k in availability_dict}
)
return check_dict.vfilter(lambda v: v < 0)
def from_file(cls, path):
data = SuperDict(suppliers=dict(sheet_name="suppliers_L1",
index_col=[0, 1, 2]),
products=dict(sheet_name="products",
index_col=[0, 1, 2]),
clients=dict(sheet_name="clients", index_col=[0, 1]),
warehouses=dict(sheet_name="warehouses",
index_col=[0, 1, 2, 3]),
distances=dict(sheet_name="distances",
index_col=[0, 1, 2]),
restricted_flows=dict(sheet_name="not_allowed_flows",
index_col=[0, 1]))
def read_table(**kwargs):
return pd.read_excel(filename=path, header=0,
**kwargs).index.values.tolist()
return cls(data.vapply(lambda v: read_table(**v)))
def check_shift_03(self, shift, id_shift):
"""
returns {"wrong_index": {(id_shift, id_location): 1, ... } for non existent locations,
"setup_time": {(id_shift, id_location): 1, ... } for operation that does not respect setup time }
"""
check = SuperDict(wrong_index=SuperDict(), setup_time=SuperDict())
id_operation = 1
for operation in shift["route"][:-1]:
if not self.is_valid_location(operation["location"]):
check["wrong_index"][(id_shift, id_operation)] = 1
continue
location = operation["location"]
if location > 0:
setup_time = self.instance.get_location_property(
location, "setupTime")
if operation["departure"] < operation["arrival"] + setup_time:
check["setup_time"][(id_shift, location)] = 1
id_operation += 1
return check
def setUp(self):
super().setUp()
self.full_inst_path = self._get_path("./data/instance.json")
self.full_inst = SuperDict.from_dict(
self.import_schema(self.full_inst_path))
# Removing parameter tables
self.full_inst["properties"] = self.full_inst["properties"].vfilter(
lambda v: v["type"] == "array")
self.one_tab_inst_path = self._get_path("./data/one_table.json")
self.one_tab_inst = SuperDict.from_dict(
self.import_schema(self.one_tab_inst_path))
self.app_name = "test"
self.second_app_name = "test_sec"
self.default_output_path = self._get_path("./data/output")
self.other_output_path = self._get_path("./data/output_path")
self.last_path = self.default_output_path
self.all_methods = TupList(
["getOne", "getAll", "deleteOne", "deleteAll", "update", "post"])
def check_solution(self, *args, **kwargs) -> dict:
return SuperDict(
availability=self.check_availability(),
restricted_flows=self.check_restricted_flows(),
demand=self.check_demand(),
second_doses=self.check_second_dose(),
warehouse_capacities=self.check_warehouse_capacity(),
consistency_warehouses=self.check_consistency_warehouses(),
consistency_suppliers=self.check_consistency_suppliers(),
)
def from_dict(cls, data_dict):
rows = SuperDict(
{
(el["id_shift"], el["position"]): SuperDict(el)
for el in data_dict["details_shifts"]
}
)
details = rows.to_dictdict().vapply(
lambda v: v.values_tl().sorted(key=lambda x: x["position"])
)
data = {
shift["id_shift"]: SuperDict(shift) for shift in data_dict["info_shifts"]
}
for shift in data:
data[shift]["route"] = details[shift]
return cls(data)
def solve(self, options: dict):
distance = (
self.instance.get_arcs()
.to_dict(result_col=["w"], indices=["n1", "n2"], is_list=False)
.kfilter(lambda k: k[0] != k[1])
)
model = cp_model.CpModel()
create_literal = lambda i, j: model.NewBoolVar("%i follows %i" % (j, i))
literals = distance.kapply(lambda k: create_literal(*k))
arcs = literals.to_tuplist()
model.AddCircuit(arcs)
model.Minimize(sum((literals * distance).values()))
solver = cp_model.CpSolver()
if options.get("msg", False):
solver.parameters.log_search_progress = True
# To benefit from the linearization of the circuit constraint.
solver.parameters.linearization_level = 2
solver.parameters.max_time_in_seconds = options.get("timeLimit", 10)
if "threads" in options:
solver.parameters.num_search_workers = options["threads"]
status = solver.Solve(model)
if options.get("msg", False):
print(solver.ResponseStats())
status_conv = {
cp_model.OPTIMAL: STATUS_OPTIMAL,
cp_model.INFEASIBLE: STATUS_INFEASIBLE,
cp_model.UNKNOWN: STATUS_UNDEFINED,
cp_model.MODEL_INVALID: STATUS_UNDEFINED,
}
if status not in [cp_model.OPTIMAL, cp_model.FEASIBLE]:
return dict(
status=status_conv.get(status), status_sol=SOLUTION_STATUS_INFEASIBLE
)
next = (
literals.vapply(solver.BooleanValue)
.vfilter(lambda v: v)
.keys_tl()
.to_dict(1, is_list=False)
)
first = next.keys_tl(0)
current_node = first
solution = TupList([first])
while True:
current_node = next[current_node]
if current_node == first:
break
solution.append(current_node)
nodes = solution.kvapply(lambda k, v: SuperDict(pos=k, node=v))
self.solution = Solution(dict(route=nodes))
return dict(status=status_conv.get(status), status_sol=SOLUTION_STATUS_FEASIBLE)
def get_demand(self) -> SuperDict:
"""
Returns a SuperDict indexed by the time slot (string) and the demand as value
For example: {"2021-09-06T07:00": 10, "2021-09-06T08:00": 15, ...}
"""
return SuperDict(
{
self._get_time_slot_string(ts): self._filter_demand(ts)
for ts in self.time_slots
}
)
def get_total_number_cut_products(self) -> SuperDict:
"""
Returns a SuperDict the products as keys and the total number of cut products as values.
For example, {'pr1': 600, 'pr2': 500, ...}
"""
number_patterns = self.get_number_bars_patterns()
number_products_per_pattern = self.get_number_products_per_pattern()
total_number_cut_products = SuperDict()
for (bar, pattern, product) in number_products_per_pattern.keys_tl():
if product not in total_number_cut_products.keys():
total_number_cut_products[product] = (
number_products_per_pattern[(bar, pattern, product)]
* number_patterns[(bar, pattern)]
)
else:
total_number_cut_products[product] += (
number_products_per_pattern[(bar, pattern, product)]
* number_patterns[(bar, pattern)]
)
return total_number_cut_products
def check_shift_05(self, shift, id_shift, trailer):
"""
returns {(shift_id, location): 1, ... } for incompatible shift-location combination
"""
check = SuperDict()
for operation in shift["route"][:-1]:
location = operation["location"]
if self.is_customer_or_source(location):
if trailer not in self.instance.get_location_property(
location, "allowedTrailers"):
check[(id_shift, location)] = 1
return check
def create_variables(self):
self.works = pl.LpVariable.dicts(
"works",
self.employee_ts_availability,
lowBound=0,
upBound=1,
cat=pl.LpBinary,
)
self.works = SuperDict(self.works)
self.starts = pl.LpVariable.dicts(
"starts",
self.employee_ts_availability,
lowBound=0,
upBound=1,
cat=pl.LpBinary,
)
self.starts = SuperDict(self.starts)
def check_shift_11(self, shift, id_shift):
"""
returns {(shift_id, location): quantity, ... } for operation with wrong sign
"""
check = SuperDict()
for i in range(1, len(shift["route"]) + 1):
operation = shift["route"][i - 1]
location = operation["location"]
if self.is_source(location) and operation["quantity"] < 0:
check[(id_shift, location)] = operation["quantity"]
elif self.is_customer(location) and operation["quantity"] > 0:
check[(id_shift, location)] = operation["quantity"]
return check