def customers_to_jsonschema(customers_data: SuperDict): customersTimeWindows = [ SuperDict(id_customer=key, **tw) for key, time_windows in customers_data.get_property("timewindows").items() for tw in time_windows ] orders = [ SuperDict(id_customer=key, **order) for key, orders in customers_data.get_property("orders").items() for order in orders ] forecasts = [ SuperDict(id_customer=key, time=time, forecast=forecast) for key, forecasts in customers_data.get_property("Forecast").items() for time, forecast in enumerate(forecasts) if forecast > 0 ] allowedTrailers = [ SuperDict(id_location=key, id_trailer=id_trailer) for key, trailers in customers_data.get_property("allowedTrailers").items() for id_trailer in trailers ] customers = drop_props_get_values( customers_data, ["timewindows", "Forecast", "allowedTrailers", "orders"]) return SuperDict( customers=customers, customersTimeWindows=customersTimeWindows, orders=orders, forecasts=forecasts, allowedTrailers=allowedTrailers, )
def check_shift_16(self, shift, id_shift): """ returns {"exceeds": {(id_shift, id_location): quantity, ... } for operation with quantity above trailer's capacity "too_low": {(id_shift, id_location): quantity, ... } for operation with quantity < min_quantity } """ check = SuperDict(exceeds=SuperDict(), too_low=SuperDict()) for i in range(1, len(shift["route"]) + 1): operation = shift["route"][i - 1] location = operation["location"] if not self.is_customer(location): continue call_in = self.instance.get_customer_property(location, "callIn") if call_in == 0: capacity = self.instance.get_customer_property( location, "Capacity") min_ope_quantity = self.instance.get_customer(location).get( "MinOperationQuantity", 0) if -operation["quantity"] > capacity: check["exceeds"][(id_shift, location)] = operation["quantity"] elif -operation["quantity"] < min_ope_quantity: check["too_low"][(id_shift, location)] = operation["quantity"] check = check.vfilter(lambda v: len(v) != 0) return check
def read_excel(path: str, param_tables_names: list = None) -> dict: """ Read an entire excel file. :param path: path of the excel file :param param_tables_names: names of the parameter tables :return: a dict with a list of dict (records format) for each table. """ is_xl_type(path) try: import openpyxl except (ModuleNotFoundError, ImportError) as e: raise Exception("You must install openpyxl package to use this method") data = pd.read_excel(path, sheet_name=None) data_tables = { name: TupList(content.to_dict(orient="records")).vapply( lambda v: SuperDict(v).vapply(lambda vv: format_value(vv))) for name, content in data.items() if name not in param_tables_names } parameters_tables = { t: SuperDict(read_param_table(path, t)).vapply(lambda v: format_value(v)) for t in param_tables_names } return {**data_tables, **parameters_tables}
def sources_to_jsonschema(sources_data): allowedTrailers = [ SuperDict(id_location=key, id_trailer=id_trailer) for key, trailers in sources_data.get_property("allowedTrailers").items() for id_trailer in trailers ] sources = drop_props_get_values(sources_data, ["allowedTrailers"]) return SuperDict(sources=sources, allowedTrailers=allowedTrailers)
def check_shift_0607(self): """ returns {"negative": {(id_shift, id_location): quantity, ... } for operation resulting in negative trailer inventory "above_capacity": {(id_shift, id_location): quantity, ... } for operation resulting in excessive trailer inventory "final_inventory": {id_shift: 1, ...} for shifts with inconsistent final inventory "initial_inventory": {id_shift: 1, ...} for shifts with inconsistent initial inventory } """ check = SuperDict( negative=SuperDict(), above_capacity=SuperDict(), final_inventory=SuperDict(), initial_inventory=SuperDict(), ) last_trailer_quantity = [0] * len(self.instance.get_id_trailers()) aux_info_shift = self.get_aux_info_shift() for trailer in self.instance.get_id_trailers(): last_trailer_quantity[ trailer] = self.instance.get_trailer_property( trailer, "InitialQuantity") for shift in sorted(list(self.solution.get_all_shifts()), key=lambda x: x["departure_time"]): driver = shift.get("driver", None) trailer = shift.get("trailer", None) id_shift = shift.get("id_shift", None) if driver is None or trailer is None: return check # Constraints (06) last_quantity = shift["initial_quantity"] for operation in shift["route"]: quantity = last_quantity + operation["quantity"] location = operation["location"] if round(quantity, 2) < 0: check["negative"][(id_shift, location)] = quantity elif round(quantity, 2) > self.instance.get_trailer_property( trailer, "Capacity"): check["above_capacity"][(id_shift, location)] = quantity last_quantity = quantity # Constraints (07) if round(aux_info_shift[id_shift]["final_inventory"], 2) != round( last_quantity, 2): check["final_inventory"][id_shift] = 1 if round(last_trailer_quantity[trailer], 2) != round( shift["initial_quantity"], 2): check["initial_inventory"][id_shift] = 1 last_trailer_quantity[trailer] = last_quantity check = check.vfilter(lambda v: len(v) != 0) return check
def matrix_to_dict(matrix, key, func): matrix = TupList(matrix).vapply(lambda v: v[key]) result = SuperDict() for L1, row in enumerate(matrix): for L2, col in enumerate(row): result[L1, L2] = func(col) return result
def index_list(table, index, _list): """ indexes the table by index and returns a dictionary with _list keys """ return (TupList(table).to_dict(result_col=_list, indices=[ index ]).vapply(lambda v: v.vapply(lambda vv: SuperDict(zip(_list, vv)))))
def from_dict(cls, data_dict): # main tables trailers = _index(data_dict["trailers"]) drivers = get_drivers(data_dict) customers = get_customers(data_dict) sources = get_sources(data_dict) # dist and time matrices matrices = {(el["L1"], el["L2"]): el for el in data_dict["matrices"]} data_out = SuperDict( trailers=trailers, drivers=drivers, customers=customers, sources=sources, matrices=matrices, coordinates=data_dict.get("coordinates", []) ) # other parameters for param in ["unit", "horizon"]: data_out[param] = data_dict["parameters"][param] data_out["bases"] = data_dict["bases"][0] return cls(data_out)
def get_aux_info_shift(self): """ Returns a dictionary containing information about each shift of self.solution, like : the ending time of the shift (in minutes), the inventory of the trailer at the end of the shift, the total duration of the shift and its driving duration For example: {2: {'arrival_time': 367, 'final_inventory': 1760, 'duration': 187, 'driving_duration': 160}, 3: {'arrival_time': 512, 'final_inventory': 950, 'duration': 203, 'driving_duration': 180} } """ shifts = self.solution.get_shifts_dict() result = SuperDict() for id_shift, shift in shifts.items(): cumulated_driving_time = 0 for s, (stop, next_stop) in enumerate( zip(shift['route'], shift['route'][1:])): stop["cumulated_driving_time"] = cumulated_driving_time cumulated_driving_time += self.instance.get_time_between( stop["location"], next_stop["location"], ) shift["route"][-1][ "cumulated_driving_time"] = cumulated_driving_time result[id_shift] = dict( arrival_time=shift["route"][-1]["arrival"], final_inventory=shift["initial_quantity"] + sum([step["quantity"] for step in shift["route"]]), duration=shift["route"][-1]["arrival"] - shift["departure_time"], driving_duration=sum( self.instance.get_time_between( stop["location"], next_stop["location"]) for stop, next_stop in zip(shift["route"], shift["route"][1:]))) return result
def solve(self, options: dict) -> dict: model = pl.LpProblem("rostering", pl.LpMaximize) # Variables: self.create_variables() # Constraints: model = self.create_constraints(model) # print(model) # Solver and solve mat_solver = pl.PULP_CBC_CMD( gapRel=0.001, timeLimit=options.get("timeLimit", 240), msg=options.get("msg", False), ) status = model.solve(mat_solver) # Check status if model.sol_status not in [pl.LpSolutionIntegerFeasible, pl.LpSolutionOptimal]: return dict(status=status, status_sol=SOLUTION_STATUS_INFEASIBLE) work_assignments = ( self.works.vfilter(lambda v: pl.value(v)) .keys_tl() .vapply(lambda v: dict(id_employee=v[1], time_slot=v[0])) ) self.solution = Solution.from_dict(SuperDict(works=work_assignments)) return dict(status=status, status_sol=SOLUTION_STATUS_FEASIBLE)
def check_solution(self): return SuperDict({ **self.check_shifts(), **self.check_sites(), **self.check_resources(), **self.check_service_quality(), }).vfilter(lambda v: len(v))
def check_qs_02(self): """ returns {location: nb_run_outs, ... } for locations experiencing runouts """ check = SuperDict() nb_run_outs = 0 site_inventories = self.calculate_inventories() for site in site_inventories.keys(): site_inventory = site_inventories[site] nb_inventory_run_out = 0 if not site_inventories[site]: continue if site > self.nb_customers + self.nb_sources + 1: continue if self.is_customer(site): customer = self.instance.get_customer(site) nb_inventory_run_out += sum( 1 for i in range(self.horizon) if round(site_inventory["tank_quantity"][i], 2) < 0 if customer["callIn"] == 0) nb_run_outs += nb_inventory_run_out if nb_inventory_run_out != 0: check[site] = nb_inventory_run_out return check
def inherit(self): all_classes = set(self.parents.keys()) not_treated = set(all_classes) treated = {"db.Model"} while not_treated: for model in not_treated: parent = self.parents[model] if parent is None: treated.add(model) continue if parent not in treated: continue treated.add(model) if parent == "db.Model": continue table_name = self.model_table[model] parent_props = self.data[ self.model_table[parent]]["items"]["properties"] parent_requirements = self.data[ self.model_table[parent]]["items"]["required"] self.data[table_name]["items"]["properties"] = SuperDict( **parent_props, **self.data[table_name]["items"]["properties"]) self.data[table_name]["items"][ "required"] += parent_requirements not_treated -= treated if not self.leave_bases: self.data = self.data.vfilter(lambda v: not v.get("remove", False))
def sub_check_qs_03(self, shift, id_shift): """ returns {(shift_id, location): 1, ... } for deliveries not related to an order to callIn customer """ check = SuperDict() for operation in shift["route"][:-1]: location = operation["location"] if not self.is_valid_location(location): continue if not self.is_customer(location): continue customer = self.instance.get_customer(location) if customer["callIn"] == 0 or not customer["orders"]: continue tw_found = False ind_tw = 0 while ind_tw < len(customer["orders"]) and not tw_found: order = customer["order"][ind_tw] if order["earliestTime"] <= operation["arrival"] <= order[ "latestTime"]: tw_found = True ind_tw += 1 if not tw_found: check[(id_shift, location)] = 1 return check
def drivers_to_jsonschema(drivers_data: SuperDict): driversTimeWindows = [ dict(id_driver=key, **tw) for key, time_windows in drivers_data.get_property("timewindows").items() for tw in time_windows ] driversTrailers = [ SuperDict(id_driver=key, id_trailer=trailer) for key, trailers in drivers_data.get_property("trailer").items() for trailer in trailers ] drivers = drop_props_get_values(drivers_data, ["trailer", "timewindows"]) return SuperDict( drivers=drivers, driversTrailers=driversTrailers, driversTimeWindows=driversTimeWindows, )
def __init__(self, data: dict): super().__init__(data) # Stores a list of the starting date of each week ordered self.weeks = TupList() # First object stores a list of the dates ordered, # the second the properties for each date. self.dates = TupList() self.dates_properties = SuperDict() # First object stores a list of the time slots ordered, # the second the properties for each one. self.time_slots = TupList() self.time_slots_properties = SuperDict() self.cache_properties()
def from_dict(cls, data: dict) -> "Solution": data_p = { el: {(v["id_employee"], v["time_slot"]): v for v in data[el]} for el in ["works"] } return cls(SuperDict(data_p))
def solve_one_iteration(self, solver, used_routes, previous_value, current_round): if 0 < current_round <= self.limit_artificial_round + 1: self.generate_new_routes() self.artificial_quantities = dict() old_used_routes = pickle.loads(pickle.dumps(used_routes, -1)) previous_routes_infos = [ (shift["id_shift"], shift["trailer"], shift["driver"]) for shift in self.solution.get_all_shifts() ] if current_round > 0: selected_routes = self.select_routes(self.nb_routes) used_routes = SuperDict({**used_routes, **selected_routes}) self.initialize_parameters(used_routes) model = self.new_model( used_routes, previous_routes_infos, current_round <= self.limit_artificial_round, ) status = model.solve(solver=solver) if status == 1: self.to_solution(model, used_routes, current_round) if current_round > self.limit_artificial_round: self.check_and_save(current_round) if status != 1 or current_round == 0: used_routes = old_used_routes return used_routes, None if not ( previous_value is None or pl.value(model.objective) < previous_value or ( (current_round > self.limit_artificial_round) and (self.last_solution_round <= self.limit_artificial_round) ) ): return old_used_routes, previous_value keep = ( self.route_var.vfilter(lambda v: pl.value(v) > 0.5) .keys_tl() .take(0) .to_dict(None) .vapply(lambda v: True) ) used_routes = { r: route for r, route in used_routes.items() if keep.get(r, False) } if current_round > self.limit_artificial_round: previous_value = pl.value(model.objective) self.last_solution_round = current_round return used_routes, previous_value
def check_shift_03(self, shift, id_shift): """ returns {"wrong_index": {(id_shift, id_location): 1, ... } for non existent locations, "setup_time": {(id_shift, id_location): 1, ... } for operation that does not respect setup time } """ check = SuperDict(wrong_index=SuperDict(), setup_time=SuperDict()) id_operation = 1 for operation in shift["route"][:-1]: if not self.is_valid_location(operation["location"]): check["wrong_index"][(id_shift, id_operation)] = 1 continue location = operation["location"] if location > 0: setup_time = self.instance.get_location_property( location, "setupTime") if operation["departure"] < operation["arrival"] + setup_time: check["setup_time"][(id_shift, location)] = 1 id_operation += 1 return check
def check_solution(self, *args, **kwargs) -> dict: return SuperDict( availability=self.check_availability(), restricted_flows=self.check_restricted_flows(), demand=self.check_demand(), second_doses=self.check_second_dose(), warehouse_capacities=self.check_warehouse_capacity(), consistency_warehouses=self.check_consistency_warehouses(), consistency_suppliers=self.check_consistency_suppliers(), )
def from_dict(cls, data_dict): rows = SuperDict( { (el["id_shift"], el["position"]): SuperDict(el) for el in data_dict["details_shifts"] } ) details = rows.to_dictdict().vapply( lambda v: v.values_tl().sorted(key=lambda x: x["position"]) ) data = { shift["id_shift"]: SuperDict(shift) for shift in data_dict["info_shifts"] } for shift in data: data[shift]["route"] = details[shift] return cls(data)
def get_demand(self) -> SuperDict: """ Returns a SuperDict indexed by the time slot (string) and the demand as value For example: {"2021-09-06T07:00": 10, "2021-09-06T08:00": 15, ...} """ return SuperDict( { self._get_time_slot_string(ts): self._filter_demand(ts) for ts in self.time_slots } )
def solve(self, options: dict): distance = ( self.instance.get_arcs() .to_dict(result_col=["w"], indices=["n1", "n2"], is_list=False) .kfilter(lambda k: k[0] != k[1]) ) model = cp_model.CpModel() create_literal = lambda i, j: model.NewBoolVar("%i follows %i" % (j, i)) literals = distance.kapply(lambda k: create_literal(*k)) arcs = literals.to_tuplist() model.AddCircuit(arcs) model.Minimize(sum((literals * distance).values())) solver = cp_model.CpSolver() if options.get("msg", False): solver.parameters.log_search_progress = True # To benefit from the linearization of the circuit constraint. solver.parameters.linearization_level = 2 solver.parameters.max_time_in_seconds = options.get("timeLimit", 10) if "threads" in options: solver.parameters.num_search_workers = options["threads"] status = solver.Solve(model) if options.get("msg", False): print(solver.ResponseStats()) status_conv = { cp_model.OPTIMAL: STATUS_OPTIMAL, cp_model.INFEASIBLE: STATUS_INFEASIBLE, cp_model.UNKNOWN: STATUS_UNDEFINED, cp_model.MODEL_INVALID: STATUS_UNDEFINED, } if status not in [cp_model.OPTIMAL, cp_model.FEASIBLE]: return dict( status=status_conv.get(status), status_sol=SOLUTION_STATUS_INFEASIBLE ) next = ( literals.vapply(solver.BooleanValue) .vfilter(lambda v: v) .keys_tl() .to_dict(1, is_list=False) ) first = next.keys_tl(0) current_node = first solution = TupList([first]) while True: current_node = next[current_node] if current_node == first: break solution.append(current_node) nodes = solution.kvapply(lambda k, v: SuperDict(pos=k, node=v)) self.solution = Solution(dict(route=nodes)) return dict(status=status_conv.get(status), status_sol=SOLUTION_STATUS_FEASIBLE)
def check_shift_05(self, shift, id_shift, trailer): """ returns {(shift_id, location): 1, ... } for incompatible shift-location combination """ check = SuperDict() for operation in shift["route"][:-1]: location = operation["location"] if self.is_customer_or_source(location): if trailer not in self.instance.get_location_property( location, "allowedTrailers"): check[(id_shift, location)] = 1 return check
def create_variables(self): self.works = pl.LpVariable.dicts( "works", self.employee_ts_availability, lowBound=0, upBound=1, cat=pl.LpBinary, ) self.works = SuperDict(self.works) self.starts = pl.LpVariable.dicts( "starts", self.employee_ts_availability, lowBound=0, upBound=1, cat=pl.LpBinary, ) self.starts = SuperDict(self.starts)
def __init__(self, instance, solution=None): super().__init__(instance, solution) self.log = "" self.solver = "MIP Model" self.routes_generator = RoutesGenerator(self.instance) self.range_hours = list(range(self.horizon)) self.routes = dict() self.unused_routes = dict() self.value_greedy = None self.interval_routes = 6 self.nb_routes = 1000 self.start_time = datetime.now() self.start_time_string = datetime.now().strftime("%d.%m-%Hh%M") self.print_log = False self.save_results = False self.artificial_quantities = dict() self.limit_artificial_round = 0 self.last_solution_round = -1 self.solution_greedy = None self.locations_in = dict() self.unique_locations_in = dict() self.hour_of_visit = dict() self.k_visit_hour = dict() self.nb_visits = dict() self.coef_inventory_conservation = 0.8 self.time_limit = 100000 # Variables self.route_var = SuperDict() self.artificial_quantities_var = SuperDict() self.artificial_binary_var = SuperDict() self.inventory_var = SuperDict() self.quantity_var = SuperDict() self.trailer_quantity_var = SuperDict()
def check_shift_11(self, shift, id_shift): """ returns {(shift_id, location): quantity, ... } for operation with wrong sign """ check = SuperDict() for i in range(1, len(shift["route"]) + 1): operation = shift["route"][i - 1] location = operation["location"] if self.is_source(location) and operation["quantity"] < 0: check[(id_shift, location)] = operation["quantity"] elif self.is_customer(location) and operation["quantity"] > 0: check[(id_shift, location)] = operation["quantity"] return check
def check_sites(self): """ returns {"site_inventory_negative": {(location, time): inventory, ... } for inventories > tank_capacity "site_inventory_exceeds": {(location, time): inventory, ... } for negative inventories "site_doesntexist": {location: 1} for nonexistent locations } """ check = SuperDict( site_inventory_negative=SuperDict(), site_inventory_exceeds=SuperDict(), site_doesntexist=SuperDict(), ) operation_quantities = [] for location in range(1 + self.nb_sources + self.nb_customers): operation_quantities.append([0] * self.horizon) site_inventories = self.calculate_inventories() for site_inventory in site_inventories.values(): location = site_inventory["location"] if not self.is_valid_location(location): check["site_doesntexist"][location] = 1 continue if location <= 1 + self.nb_sources: continue call_in = self.instance.get_customer_property(location, "callIn") if call_in == 1: continue for i in range(self.horizon): if round(site_inventory["tank_quantity"][i], 3) < 0: check["site_inventory_negative"][ location, i] = site_inventory["tank_quantity"][i] if round(site_inventory["tank_quantity"][i], 3) > self.instance.get_customer_property( location, "Capacity"): check["site_inventory_exceeds"][ location, i] = site_inventory["tank_quantity"][i] check = check.vfilter(lambda v: len(v) != 0) return check
def _get_dates_properties(self) -> SuperDict: """ Returns a SuperDict with dates as key and its dict properties as a value For example: {datetime(2021, 9, 6, 0, 0, 0): {"string": "2021-09-06", "week": 36}, ...} """ return SuperDict( { date: { "string": get_date_string_from_ts(date), "week": get_week_from_ts(date), } for date in self.dates } )
def calculate_inventories(self): """ Calculates the inventory of each customer at each hour of the time horizon :return: A dictionary whose keys are the indexes of the customers and whose values are dictionaries containing two elements: - a list 'tank_quantity' containing the value of the inventory at each hour - an integer 'location' corresponding to the index of the customer For example: {2: {'tank_inventory': [15000, 14000, 13000, 16000, ... , 14000, 13000], 'location': 2}, 3: {'tank_inventory': [4000, 4000, 4000, 1000, ..., 3000, 3000], 'location': 3}} """ customers = self.instance.get_id_customers() _get_customer = lambda c, p: self.instance.get_location_property(c, p) # we need three things: consumption, initial stock and arrivals # we store each as a dictionary where each customer has a numpy array of length = horizon # 1. we get consumptions from forecasts consumption = (customers.to_dict(None).vapply( _get_customer, "Forecast").vapply(lambda v: -np.array(v)).vapply( lambda v: v[0:self.horizon])) # 2. we get initial tanks and assign them to the first period initial_tank = consumption.vapply(lambda v: np.zeros(self.horizon)) for k, v in initial_tank.items(): v[0] = _get_customer(k, "InitialTankQuantity") # 3. we now use the solution to get the arrivals with routes shifts = self.solution.get_all_shifts() all_operations = TupList(operation for route_list in shifts.take("route") for operation in route_list) # for each route we take the location, the time and how much. # we only get customers and round the time and make positive the quantity arrivals_tup = (all_operations.take( ["location", "arrival", "quantity"]).vfilter(lambda v: self.is_customer(v[0])).vapply( lambda v: (v[0], floor(v[1] / self.unit), -round(v[2], 3)))) # we initialize at 0 and increase when a truck arrives: arrivals = consumption.vapply(lambda v: np.zeros(self.horizon)) for customer, time, quantity in arrivals_tup: arrivals[customer][time] += quantity # we take advantage of both pytups broadcasting and numpys broadcasting # then we accumulate over periods stocks = (consumption + arrivals + initial_tank).vapply(np.cumsum) site_inventories = SuperDict() for _id, quantity_arr in stocks.items(): site_inventories[_id] = dict(tank_quantity=quantity_arr, location=_id) return site_inventories