def set_default_maintenances(self): params = self.data['parameters'] if 'maint_duration' not in params: # we're using the new parameters, no need for defaults return resources = self.data['resources'] maints = { 'M': { 'duration_periods': params['maint_duration'], 'capacity_usage': 1, 'max_used_time': params['max_used_time'], 'max_elapsed_time': params['max_elapsed_time'], 'elapsed_time_size': params['elapsed_time_size'], 'used_time_size': params['max_used_time'], 'type': '1', 'capacity': params['maint_capacity'], 'depends_on': [] } } if 'maintenances' in self.data: maints = sd.SuperDict.from_dict(maints) maints.update(self.data['maintenances']) self.data['maintenances'] = sd.SuperDict.from_dict(maints) if 'initial' not in resources.values_l()[0]: data_resources = self.data['resources'] rut_init = data_resources.get_property('initial_used') ret_init = data_resources.get_property('initial_elapsed') for r in resources: self.data['resources'][r]['initial'] = \ sd.SuperDict(M=sd.SuperDict(elapsed=ret_init[r], used=rut_init[r])) return
def generate_resource_types(d_param, data_input): start_period = d_param['start'] num_parallel_tasks = d_param['num_parallel_tasks'] d_tasks = data_input['tasks'] last_period = d_param['end'] num_resources = d_param['num_resources'] resources = [str(r) for r in range(num_resources)] period_type_num_resource = { t: {p: 0 for p in aux.get_months(start_period, last_period)} for t in range(num_parallel_tasks) } for k, v in d_tasks.items(): t = v['type_resource'] for p in aux.get_months(v['start'], v['end']): period_type_num_resource[t][p] += v['num_resource'] # we want at least as many resources as the max requirement of any given month # for the rest, we randomly assign a type weighted by the needs max_res_need_type = sd.SuperDict( {k: max(v.values()) for k, v in period_type_num_resource.items()}) res_types = [t for t, q in max_res_need_type.items() for r in range(q)] res_types.extend( rn.choices(max_res_need_type.keys_l(), k=len(resources) - len(res_types), weights=max_res_need_type.values_l())) res_types = sd.SuperDict( {k: res_types[i] for i, k in enumerate(resources)}) return res_types
def get_category(self, category, param=None, default_dict=None): assert category in self.data data = self.data[category] if not data: return sd.SuperDict() if default_dict is not None: default_dict = sd.SuperDict.from_dict(default_dict) data = data.vapply(lambda v: sd.SuperDict({**default_dict, **v})) if param is None: return data if param in list(data.values())[0]: return data.get_property(param) raise IndexError("param {} is not present in the category {}".format( param, category))
def get_initial_state(self, time_type, resource=None, maint='M'): """ :param time_type: elapsed or used :param resource: optional value to filter only one resource :return: """ if time_type not in ["elapsed", "used"]: raise KeyError( "Wrong type in time_type parameter: elapsed or used only") initials = sd.SuperDict(self.get_resources('initial')) if resource is not None: initials = initials.filter(resource, check=False) return sd.SuperDict( {k: v[maint][time_type] for k, v in initials.items()})
def set_date_params(self): """ This makes a cache of periods to later access it to move between periods inside the planning horizon :return: """ start = self.get_param('start') num_periods = self.get_param('num_period') self.data['aux'] = sd.SuperDict() extra_time = num_periods // 12 * 2 + 5 begin_year = int(start[:4]) - extra_time end_year = int(start[:4]) + num_periods // 12 + extra_time many_months = [ '{}-{:02.0f}'.format(_a, _b) for _a in range(begin_year, end_year) for _b in range(1, 13) ] pos = many_months.index(start) self.data['aux']['period_e'] = { k - pos: r for k, r in enumerate(many_months) } self.data['aux']['period_i'] = { v: k for k, v in self.data['aux']['period_e'].items() } # we need to guarantee consistency between start, num_periods and end self.data['parameters']['end'] = self.shift_period( start, num_periods - 1)
def check_task_num_resources(self, deficit_only=True, assign_missions=True, periods=None, resources=None, **params): if not assign_missions: return sd.SuperDict() if periods is None: periods = self.instance.get_periods().to_set() else: periods = set(periods) task_reqs = self.instance.get_tasks('num_resource') task_period_list = \ self.instance.get_task_period_list().\ vfilter(lambda v: v[1] in periods) task_under_assigned = \ self.solution.get_task_num_resources(periods, resources).\ fill_with_default(task_period_list).\ kvapply(lambda k, v: task_reqs[k[0]] - v) if not deficit_only: return task_under_assigned else: return task_under_assigned.vfilter(lambda x: x > 0)
def generate_resource_capacites(data_input, res_types): d_tasks = data_input['tasks'] # we initialize the capacities with the types of resources res_capacities = sd.SuperDict({k: {str(v)} for k, v in res_types.items()}) # we want to add the special capacities to only some resources. # we do this by iterating over tasks and trying to complete the partially able resources. # We will start with the tasks that demand the most capacities task_content = sorted(d_tasks.items(), key=lambda x: -len(x[1]['capacities'])) for _task, contents in task_content: _capacities = set(contents['capacities']) _t_type = str(contents['type_resource']) _resources = contents['num_resource'] # we filter resources that have the task's type _res_possible = res_capacities.clean(func=lambda x: _t_type in x) # then we see if we already have matching resources: _res_ready = _res_possible.clean( func=lambda x: not _capacities.difference(x)) new_resources = _resources * 2 - len(_res_ready) if new_resources > 0: # we take new_resources random resources from _res_possible. # we try to upgrade the ones with the least capacities _res_to_upgrade = \ rn.choices(_res_possible.keys_l(), k=new_resources, weights=[1/len(v) for v in _res_possible.values_l()]) for k in _res_to_upgrade: for cap in _capacities: res_capacities[k].add(cap) return res_capacities
def get_task_num_resources(self, periods=None, resources=None): tasks = self.get_tasks() if periods: periods = set(periods) tasks = tasks.kfilter(lambda k: k[1] in periods) if resources: resources = set(resources) tasks = tasks.kfilter(lambda k: k[0] in resources) if not len(tasks): return sd.SuperDict() resource, period = zip(*tasks.keys()) task = tasks.values_l() keys, values = np.unique(np.array(list(zip(task, period))), axis=0, return_counts=True) result = sd.SuperDict({tuple(k): v for k, v in zip(keys, values)}) return result
def get_cluster_constraints(self): result = self.data['aux'].get('cluster_constraints') if result: return result min_percent = self.get_param('min_avail_percent') min_value = self.get_param('min_avail_value') hour_perc = self.get_param('min_hours_perc') # cluster availability will now mean: # resources that are not under maintenance cluster = self.get_clusters() kt = [(c, period) for c in cluster.values() for period in self.get_periods()] num_res_maint = \ sd.SuperDict(self.get_fixed_maintenances_cluster()).\ to_lendict().\ fill_with_default(keys=kt) c_num_candidates = sd.SuperDict( self.get_cluster_candidates()).to_lendict() c_slack = { tup: c_num_candidates[tup[0]] - num_res_maint[tup] for tup in kt } c_min = { (k, t): min(c_slack[(k, t)], max(math.ceil(c_num_candidates[k] * min_percent), min_value)) for (k, t) in c_slack } # c_needs_num = {(k, t): c_num_candidates[k] - c_min[k, t] - num_res_maint[k, t] for k, t in kt} # TODO: this should depend on the maintenance (we assumme 'M') limit = self.get_maintenances('max_used_time')['M'] c_needs_hours = {(k, t): v * limit * hour_perc for k, v in c_num_candidates.items() for t in self.get_periods()} result = self.data['aux'][ 'cluster_constraints'] = sd.SuperDict.from_dict({ 'num': c_needs_num, 'hours': c_needs_hours }) return result
def get_cluster_candidates(self, resource=None): # Since clusters are strict, their candidates are the same as the tasks. c_candidates = sd.SuperDict() t_candidates = self.get_task_candidates(resource=resource) cluster = self.get_clusters() for k, v in t_candidates.items(): c_candidates[cluster[k]] = v return c_candidates
def get_fixed_maintenances_cluster(self): fixed_per_period = self.get_fixed_maintenances(dict_key='period') candidates_per_cluster = self.get_cluster_candidates() fixed_per_period_cluster = sd.SuperDict() for period, resources in fixed_per_period.items(): for cluster, candidates in candidates_per_cluster.items(): fixed_per_period_cluster[(cluster, period)] =\ np.intersect1d(resources, candidates) return fixed_per_period_cluster
def prepare_maint_params(self): maints = self.data['maintenances'] depends_on = sd.SuperDict(maints).get_property('depends_on') for m in depends_on: if m not in maints[m]['depends_on']: maints[m]['depends_on'].append(m) affects = depends_on.list_reverse() for m, v in affects.items(): maints[m]['affects'] = v return maints
def check_resource_in_candidates(self, **params): task_solution = self.solution.get_tasks() if not len(task_solution): return sd.SuperDict() task_candidates = self.instance.get_task_candidates() bad_assignment = {(resource, period): task for (resource, period), task in task_solution.items() if resource not in task_candidates[task]} return sd.SuperDict.from_dict(bad_assignment)
def check_fixed_assignments(self, **params): first_period = self.instance.get_param('start') last_period = self.instance.get_param('end') state_tasks = self.solution.get_state_tasks().to_list() fixed_states = self.instance.get_fixed_states() fixed_states_h = \ fixed_states.\ vfilter(lambda x: first_period <= x[2] <= last_period).\ take([0, 2, 1]) diff_tups = set(fixed_states_h) - set(state_tasks) return sd.SuperDict({k: 1 for k in diff_tups})
def migrate_to_multimaint(self): states = self.data.get('state') if not states: return # here, we have an old solution format # we just convert the maint into a dict of maints self.data['state_m'] = \ states.to_dictup().\ vapply(lambda v: sd.SuperDict({v: 1})).\ to_dictdict() self.data.pop('state') return
def check_resource_state(self, **params): task_solution = self.solution.get_tasks() state_solution = self.solution.get_state_tuplist().take([0, 1]) task_solution_k = np.fromiter(task_solution.keys(), dtype=[('A', '<U6'), ('T', 'U7')]) state_solution_k = np.asarray(state_solution, dtype=[('A', '<U6'), ('T', 'U7')]) duplicated_states = \ np.intersect1d(task_solution_k, state_solution_k) return sd.SuperDict({tuple(item): 1 for item in duplicated_states})
def generate_tasks(data_input): d_param = data_input['parameters'] last_period = d_param['end'] num_parallel_tasks = d_param['num_parallel_tasks'] start_period = d_param['start'] t_min_assign = d_param['t_min_assign'] t_required_hours = d_param['t_required_hours'] t_num_resource = d_param['t_num_resource'] t_duration = d_param['t_duration'] perc_add_capacity = d_param['perc_add_capacity'] # Here we simulate the tasks along the planning horizon. # we need to guarantee there are num_parallel_tasks active task # at each time. task = 0 t_start = {} t_end = {} t_type = {} for t in range(num_parallel_tasks): date = start_period while date <= last_period: t_type[task] = t t_start[task] = date date = aux.shift_month(date, rn.randint(*t_duration)) if date > last_period: date = aux.get_next_month(last_period) t_end[task] = aux.get_prev_month(date) task += 1 t_capacites = {k: {str(v)} for k, v in t_type.items()} optionals = list(st.ascii_uppercase)[::-1] for _task in t_capacites: if rn.random() < perc_add_capacity: t_capacites[_task].add(optionals.pop()) return sd.SuperDict({ str(t): { 'start': t_start[t], 'end': t_end[t], 'consumption': math.floor(np.random.triangular(*t_required_hours)), 'num_resource': rn.randint(*t_num_resource), 'type_resource': t_type[t], 'matricule': '' # this is aesthetic , 'min_assign': rn.choice(t_min_assign), 'capacities': list(t_capacites[t]) } for t in t_start })
def set_start_periods(self): """ This function remakes the start tasks assignments and states (maintenances) It edits the aux part of the solution examples :return: the dictionary that it assigns """ tasks_start = self.get_task_periods() states_start = self.get_state_periods() all_starts = tasks_start + states_start starts = {(r, t): v for (r, t, v, _) in all_starts} if 'aux' not in self.solution.data: self.solution.data['aux'] = sd.SuperDict() self.solution.data['aux']['start'] = sd.SuperDict.from_dict(starts) return starts
def gerenate_initial_assignments(d_param, data_input, res_in_maint, res_capacities): num_resources = d_param['num_resources'] resources = [str(r) for r in range(num_resources)] start_period = d_param['start'] d_tasks = data_input['tasks'] # Secondly, for the remaining resources, we assign the previous tasks # at the beginning of the planning horizon _res = [r for r in resources if r not in res_in_maint] res_task_init = {} starting_tasks = [ k for k, v in d_tasks.items() if v['start'] == start_period ] for j in starting_tasks: capacities = set(d_tasks[j]['capacities']) # first we choose a number of resources equivalent to the number # of resources needed by the task # these resources need to be able to do the task _res_task = [ r for r in _res if not capacities.difference(res_capacities[r]) ] res_to_assign = \ np.random.choice(_res_task, # min(rn.randrange(d_tasks[j]['num_resource'] + 1), len(_res)), min(d_tasks[j]['num_resource'], len(_res)), replace=False) # we take them out of the list of available resources: _res = [r for r in _res if r not in res_to_assign] # now we decide when was the task assigned: min_assign = d_tasks[j]['min_assign'] for r in res_to_assign: # we assumed the resource could have had the task # for a time = double the minimum time months_in_task = rn.randrange(min_assign) * 2 # if the resource started less than min_assin ago, # we fix the task for the following periods res_task_init[r] = { aux.shift_month(start_period, -n - 1): j for n in range(months_in_task) } if not _res: break res_task_init = sd.SuperDict(res_task_init).fill_with_default(resources, default={}) return res_task_init
def check_min_flight_hours_seminew(self, recalculate=True, deficit_only=True, periods=None, **params): if recalculate: ruts = self.set_remaining_usage_time(time='rut', maint='M') else: ruts = self.get_remainingtime(time='rut', maint='M') all_periods = self.instance.get_periods().to_set() if periods is None: periods = all_periods else: periods = set(periods) & all_periods cluster_data = self.instance.get_cluster_constraints() min_hours = cluster_data['hours'] resources = self.instance.get_resources().keys_tl().vapply( int).sorted() clusters = \ self.instance.get_cluster_candidates(). \ vapply(lambda v: set(int(vv) for vv in v)). \ vapply(lambda v: [r in v for r in resources]). \ vapply(np.array) positions = self.instance.get_period_positions() ruts_dt = ruts.to_dictup() res_arr, periods_arr = zip(*ruts_dt.keys()) rut_arr = ruts_dt.values_l() res_arr = np.array(res_arr, dtype='int') periods_arr = np.array([positions[p] for p in periods_arr]) rut_arr = np.array(rut_arr) def deficit(c, p, candidates): pos = positions[p] mask = (candidates[res_arr]) & (periods_arr == pos) return np.sum(rut_arr[mask]) - min_hours[c, p] hours_deficit = \ sd.SuperDict({ (c, p): deficit(c, p, candidates) for c, candidates in clusters.items() for p in periods }) if deficit_only: hours_deficit = hours_deficit.vfilter(lambda x: x < 0) return hours_deficit
def get_acc_consumption(self): _range = self.instance.get_periods_range _dist = self.instance.get_dist_periods _prev = self.instance.get_prev_period maint_cycle = self.get_all_maintenance_cycles() rut = sd.SuperDict.from_dict(self.set_remaining_usage_time('rut')) rem_hours_cycle = sd.SuperDict() for k, cycles in maint_cycle.items(): for pos, (start, stop) in enumerate(cycles): limit = rut[k][_prev( start)] # should be initial_rut or max_rut _periods = _range(start, stop) rem_hours_cycle[k, start, stop] = \ limit * (_dist(start, stop) + 1) - \ sum(rut[k].filter(_periods).values()) return rem_hours_cycle
def get_instances_paths(self): num_slashes = 2 keys_positions = [1, 2] if self.no_scenario: num_slashes = 1 keys_positions = 1 zipobj = zipfile.ZipFile(self.path) all_files = tl.TupList(di.dirs_in_zip(zipobj)) scenario_instances = all_files.vfilter( lambda f: f.count("/") == num_slashes) keys = (scenario_instances.vapply( str.split, "/").vapply(tuple).take(keys_positions)) result_dict = sd.SuperDict(zip(keys, scenario_instances)) if self.scenarios: scenarios = set(self.scenarios) return result_dict.kfilter(lambda k: k[0] in scenarios) else: return result_dict
def check_solution(self, list_tests=None, **params): func_list = { 'candidates': self.check_resource_in_candidates, 'state': self.check_resource_state, 'resources': self.check_task_num_resources, 'usage': self.check_usage_consumption, 'elapsed': self.check_elapsed_consumption, 'min_assign': self.check_min_max_assignment, 'available': self.check_min_available, 'hours': self.check_min_flight_hours, 'start_periods': self.check_fixed_assignments, 'dist_maints': self.check_min_distance_maints, 'capacity': self.check_sub_maintenance_capacity, 'maint_size': self.check_maints_size } if list_tests is None: list_tests = func_list.keys() result = {k: func_list[k](**params) for k in list_tests} return sd.SuperDict({k: v for k, v in result.items() if v})
def check_min_flight_hours(self, recalculate=True, deficit_only=True, periods=None, resources=None, **params): """ :param recalculate: recalculate ruts (not use cache) :param deficit_only: return all, not only failed checks :param periods: optional filter for periods to check :param resources: optional filter for resources to count :param params: for compatibility :return: """ if recalculate: ruts = self.set_remaining_usage_time(time='rut', maint='M') else: ruts = self.get_remainingtime(time='rut', maint='M') if resources is not None: ruts = ruts.filter(resources) all_periods = self.instance.get_periods().to_set() if periods is None: periods = all_periods else: periods = set(periods) & all_periods cluster_data = self.instance.get_cluster_constraints() min_hours = cluster_data['hours'] clusters = self.instance.get_cluster_candidates().list_reverse() ruts_dt = ruts.to_dictup() data = [((c, p), h) for (r, p), h in ruts_dt.items() for c in clusters[r] if p in periods] keys, weights = zip(*data) dict_keys = min_hours.keys_tl() equiv = {k: pos for pos, k in enumerate(dict_keys)} keys_int = np.array([equiv[k] for k in keys]) dict_values = np.bincount(keys_int, weights=weights) hours_deficit2 = sd.SuperDict( {k: v - min_hours[k] for k, v in zip(dict_keys, dict_values)}) if deficit_only: hours_deficit2 = hours_deficit2.vfilter(lambda x: x < 0) return hours_deficit2
def correct_initial_state(self, time_type): """ Returns the correct initial states for resources. It corrects it using the max and whether it is in maintenance. :param time_type: :return: """ first_period = self.get_param('start') prev_first_period = self.get_prev_period(first_period) if time_type not in ["elapsed", "used"]: raise KeyError( "Wrong type in time_type parameter: elapsed or used only") key_initial = "initial_" + time_type key_max = "max_" + time_type + "_time" resources = sd.SuperDict(self.get_resources()) res1 = resources.keys_l()[0] if key_initial not in res1: # this means we're already using the good nomenclature return rt_read = resources.get_property(key_initial) rt_max = self.get_param(key_max) # here, we calculate the number of fixed maintenances. res_maints = \ self.get_fixed_maintenances().\ vfilter(lambda x: x[1] >= prev_first_period).\ to_dict(result_col=1).\ to_lendict() if time_type == 'elapsed': # this extra is only for ret, not for rut rt_fixed = {k: rt_max + v - 1 for k, v in res_maints.items()} else: rt_fixed = {k: rt_max for k, v in res_maints.items()} rt_init = dict(rt_read) rt_init.update(rt_fixed) for r in rt_init: self.data['resources'][r]['M'][key_initial] = rt_init[r] return rt_init
def check_resource_consumption(self, time='rut', recalculate=True, min_value=0, **params): """ This function (calculates and) checks the "remaining time" for all maintenances :param time: calculate rut or ret :param recalculate: used cached rut and ret :param params: optional. compatibility :return: {(maint, resource, period): remaining time} """ if recalculate: rt_maint = self.set_remaining_usage_time_all(time=time) else: rt_maint = self.solution.data['aux'][time] return sd.SuperDict(rt_maint).to_dictup().\ clean(func=lambda x: x is not None and x <= min_value)
def get_status(self, candidate): """ This function is great for debugging :param candidate: a resource :return: dataframe with everything that's going on with the resource """ data = self.solution.data if 'aux' not in data: data['aux'] = sd.SuperDict() for t in ['rut', 'ret']: if t not in data['aux']: self.set_remaining_usage_time_all(time=t) if 'start' not in data['aux']: self.set_start_periods() data_maints = self.instance.get_maintenances() _rut = { m: data['aux']['rut'][m].get(candidate, {}) for m in self.instance.get_rt_maints('rut') } rut = pd.DataFrame.from_dict(_rut) _ret = { m: data['aux']['ret'][m].get(candidate, {}) for m in self.instance.get_rt_maints('ret') } ret = pd.DataFrame.from_dict(_ret) # ret = pd.DataFrame.from_dict(examples['aux']['ret']['M'].get(candidate, {}), orient='index') start = pd.DataFrame.from_dict(data['aux']['start'].get(candidate, {}), orient='index') # state = pd.DataFrame.from_dict(examples['state'].get(candidate, {}), orient='index') state_m = pd.DataFrame.from_dict(data['state_m'].get(candidate, {}), orient='index') task = pd.DataFrame.from_dict(data['task'].get(candidate, {}), orient='index') args = {'left_index': True, 'right_index': True, 'how': 'left'} table = rut.merge(ret, **args).merge(task, **args).\ merge(start, **args).sort_index().merge(state_m, **args) names = np.all(table.isna(), axis=0) list_names = names[~names].index table = table.filter(list_names) return table.reset_index().rename(columns={'index': 'period'})
def check_min_max_assignment(self, **params): """ :return: periods were the min assignment (including maintenance) in format: (resource, start, end): error. if error negative: bigger than max. Otherwise: less than min is not respected """ # TODO: do it with self.solution.get_schedule() tasks = self.solution.get_tasks().to_tuplist() maints = self.solution.get_state_tuplist() previous = sd.SuperDict.from_dict(self.instance.get_resources("states")).\ to_dictup().to_tuplist() min_assign = self.instance.get_min_assign() max_assign = self.instance.get_max_assign() num_periods = self.instance.get_param('num_period') ct = self.instance.compare_tups all_states = maints + tasks + previous all_states_periods = \ tl.TupList(all_states).\ sorted(key=lambda v: (v[0], v[2], v[1])).\ to_start_finish(ct, sort=False) first_period = self.instance.get_param('start') last_period = self.instance.get_param('end') incorrect = {} for (resource, start, state, finish) in all_states_periods: # periods that finish before the horizon # or at the end are not checked if finish < first_period or finish == last_period: continue size_period = len(self.instance.get_periods_range(start, finish)) if size_period < min_assign.get(state, 1): incorrect[resource, start, finish, state] = min_assign[state] - size_period elif size_period > max_assign.get(state, num_periods): incorrect[resource, start, finish, state] = max_assign[state] - size_period return sd.SuperDict(incorrect)
def generate_resources_in_maint(d_param): # Here we simulate the initial state of resources. # First of all we decide which resources are in maintenance # and then the periods they have been under maintenance start_period = d_param['start'] maint_duration = d_param['maint_duration'] num_resources = d_param['num_resources'] resources = [str(r) for r in range(num_resources)] perc_in_maint = d_param['perc_in_maint'] res_in_maint = np.random.choice(resources, math.floor(num_resources * perc_in_maint), replace=False) res_maint_init = { r: { aux.shift_month(start_period, -n - 1): 'M' for n in range(rn.randrange(maint_duration) + 1) } for r in res_in_maint } res_maint_init = sd.SuperDict(res_maint_init).\ fill_with_default(resources, default={}) return res_in_maint, res_maint_init
def get_max_assign(self): return sd.SuperDict(self.get_maintenances('duration_periods'))