def solve(self):
T = self.g.size
with self.profiler('Gurobi model optimization',
extra_data={
'T': str(T),
'budget': str(self.budget)
}):
with Timer("ILPSolve") as solve_ilp:
self.m.optimize()
self.solve_time = solve_ilp.elapsed
infeasible = (self.m.status == GRB.INFEASIBLE)
try:
_ = self.R[0, 0].X
_ = self.S[0, 0].X
_ = self.U[0, 0].X
_ = self.batch_size.X
except AttributeError as e:
infeasible = True
if infeasible:
raise ValueError(
"Infeasible model, check constraints carefully. Insufficient memory?"
)
Rout = np.zeros((T, T), dtype=SOLVER_DTYPE)
Sout = np.zeros((T, T), dtype=SOLVER_DTYPE)
Uout = np.zeros((T, T), dtype=SOLVER_DTYPE)
Free_Eout = np.zeros((T, len(self.g.edge_list)), dtype=SOLVER_DTYPE)
batch_size = self.batch_size.X
try:
for t in range(T):
for i in range(T):
Rout[t][i] = int(self.R[t, i].X)
Sout[t][i] = int(self.S[t, i].X)
Uout[t][i] = self.U[t, i].X * self.ram_gcd
for e in range(len(self.g.edge_list)):
Free_Eout[t][e] = int(self.Free_E[t, e].X)
except AttributeError as e:
logging.exception(e)
return None, None, None, None
Rout = solve_r_opt(self.g,
Sout) # prune R using optimal recomputation solver
ilp_aux_data = ILPAuxData(U=Uout,
Free_E=Free_Eout,
ilp_approx=False,
ilp_time_limit=0,
ilp_eps_noise=0,
ilp_num_constraints=self.m.numConstrs,
ilp_num_variables=self.m.numVars)
schedule, aux_data = schedule_from_rs(self.g, Rout, Sout)
return ScheduledResult(solve_strategy=SolveStrategy.OPTIMAL_ILP_GC,
solver_budget=self.budget,
feasible=True,
schedule=schedule,
schedule_aux_data=aux_data,
solve_time_s=self.solve_time,
ilp_aux_data=ilp_aux_data), batch_size
def read_result(self, cache_key: RedisCacheKey, ilp_time_limit: int = -1) -> Optional[ScheduledResult]:
key = cache_key.key(ilp_time_limit)
result_bytes = self.redis_conn.get(key)
if result_bytes:
res = ScheduledResult.loads(result_bytes)
if res.solve_strategy == SolveStrategy.OPTIMAL_ILP_GC:
if res.ilp_aux_data is not None and (res.ilp_aux_data.ilp_time_limit >= ilp_time_limit):
return res
elif res.schedule_aux_data is not None:
return res
return None
def solve_checkpoint_last_node(g: DFGraph):
"""Checkpoint only one node between stages"""
with Timer('solve_checkpoint_last_node') as timer_solve:
s = np.zeros((g.size, g.size), dtype=SOLVER_DTYPE)
np.fill_diagonal(s[1:], 1)
r = solve_r_opt(g, s)
schedule, aux_data = schedule_from_rs(g, r, s)
return ScheduledResult(solve_strategy=SolveStrategy.CHECKPOINT_LAST_NODE,
solver_budget=0,
feasible=True,
schedule=schedule,
schedule_aux_data=aux_data,
solve_time_s=timer_solve.elapsed)
def solve_checkpoint_all(g: DFGraph):
with Timer('solve_checkpoint_all') as timer_solve:
s = gen_s_matrix_fixed_checkpoints(g, g.vfwd)
r = solve_r_opt(g, s)
schedule, aux_data = schedule_from_rs(g, r, s)
return ScheduledResult(
solve_strategy=SolveStrategy.CHECKPOINT_ALL,
solver_budget=0,
feasible=True,
schedule=schedule,
schedule_aux_data=aux_data,
solve_time_s=timer_solve.elapsed
)
def solve_chen_sqrtn(g: DFGraph, use_actuation_points: bool) -> ScheduledResult:
with Timer('solve_chen_sqrtn') as timer_solve:
C = g.checkpoint_set if use_actuation_points else g.checkpoint_set_all
k = int(math.sqrt(len(C)))
checkpoints = [v for idx, v in enumerate(C) if (idx + 1) % k == 0]
S = gen_s_matrix_fixed_checkpoints(g, set(checkpoints))
R = solve_r_opt(g, S)
schedule, aux_data = schedule_from_rs(g, R, S)
return ScheduledResult(
solve_strategy=SolveStrategy.CHEN_SQRTN if use_actuation_points else SolveStrategy.CHEN_SQRTN_NOAP,
solver_budget=0,
feasible=True,
schedule=schedule,
schedule_aux_data=aux_data,
solve_time_s=timer_solve.elapsed
)
def solve_griewank(g: DFGraph, budget: int):
# todo check graph is chain graph
with Timer('solve_griewank') as timer_solve:
r, s = _solve_griewank_to_rs(g, budget)
schedule, aux_data = schedule_from_rs(g, r, s)
griewank_feasible = (r is not None
and s is not None) # griewank load from FS
return ScheduledResult(
solve_strategy=SolveStrategy.GRIEWANK_LOGN,
solver_budget=budget,
feasible=griewank_feasible,
schedule=schedule,
schedule_aux_data=aux_data,
solve_time_s=timer_solve.
elapsed # this is technically just filesystem load time
)
def solve_simrd(g: DFGraph,
budget: int,
heuristic: Heuristic,
runtime: TelemetrizedRuntimeBase,
thrash: float,
liveness=True):
simrd_g = from_dfgraph(g, liveness_analysis=liveness)
callback = simrd_g.get_closure()
base_compute = sum(g.cost_cpu.values())
assert np.isclose(base_compute, get_simrd_base_compute(callback))
remat_limit = base_compute * (thrash - 1)
rt = runtime(budget=budget, heuristic=heuristic, remat_limit=remat_limit)
try:
with Timer("solve_simrd") as timer_solve:
callback(rt)
assert rt.clock - 1 >= base_compute
assert np.isclose(
rt.clock - 1, rt.telemetry.summary['model_compute'] +
rt.telemetry.summary['remat_compute'])
feasible = True
except (MemoryError, RematExceededError):
feasible = False
# TODO: build schedule from execution
if feasible:
aux_data = SchedulerAuxData(
R=None,
S=None,
cpu=rt.clock - 1,
peak_ram=rt.telemetry.summary['max_memory'] + g.cost_ram_fixed,
activation_ram=rt.telemetry.summary['max_memory'],
mem_grid=None,
mem_timeline=None,
schedule_time_s=None)
else:
aux_data = None
return ScheduledResult(solve_strategy=SolveStrategy.SIMRD,
solver_budget=budget,
feasible=feasible,
schedule=None,
schedule_aux_data=aux_data,
ilp_aux_data=None,
solve_time_s=timer_solve.elapsed)
def solve_chen_greedy(g: DFGraph, segment_mem_B: int, use_actuation_points: bool):
with Timer('solve_chen_greedy') as timer_solve:
C = g.checkpoint_set if use_actuation_points else g.checkpoint_set_all
temp = 0
x = 0
checkpoints = set()
for v in g.topological_order_fwd:
temp += g.cost_ram[v]
if v in C and temp > segment_mem_B:
x += g.cost_ram[v]
temp = 0
checkpoints.add(v)
S = gen_s_matrix_fixed_checkpoints(g, checkpoints)
R = solve_r_opt(g, S)
schedule, aux_data = schedule_from_rs(g, R, S)
return ScheduledResult(
solve_strategy=SolveStrategy.CHEN_GREEDY if use_actuation_points else SolveStrategy.CHEN_GREEDY_NOAP,
solver_budget=segment_mem_B,
feasible=True,
schedule=schedule,
schedule_aux_data=aux_data,
solve_time_s=timer_solve.elapsed
)
def solve_ilp_gurobi(
g: DFGraph,
budget: int,
seed_s: Optional[np.ndarray] = None,
approx=True,
imposed_schedule: ImposedSchedule = ImposedSchedule.FULL_SCHEDULE,
solve_r=True,
time_limit: Optional[int] = None,
write_log_file: Optional[PathLike] = None,
print_to_console=True,
write_model_file: Optional[PathLike] = None,
eps_noise=0.01,
solver_cores=os.cpu_count()):
"""
Memory-accurate solver with garbage collection.
:param g: DFGraph -- graph definition extracted from model
:param budget: int -- budget constraint for solving
:param seed_s: np.ndarray -- optional parameter to set warm-start for solver, defaults to empty S
:param approx: bool -- set true to return as soon as a solution is found that is within 1% of optimal
:param imposed_schedule -- selects a set of constraints on R and S that impose a schedule or require some nodes to be computed
:param solve_r -- if set, solve for the optimal R
:param time_limit: int -- time limit for solving in seconds
:param write_log_file: if set, log gurobi to this file
:param print_to_console: if set, print gurobi logs to the console
:param write_model_file: if set, write output model file to this location
:param eps_noise: float -- if set, inject epsilon noise into objective weights, default 0.5%
:param solver_cores: int -- if set, use this number of cores for ILP solving
"""
param_dict = {
'LogToConsole': 1 if print_to_console else 0,
'LogFile': str(write_log_file) if write_log_file is not None else "",
'Threads': solver_cores,
'TimeLimit': math.inf if time_limit is None else time_limit,
'OptimalityTol': 1e-2 if approx else 1e-4,
'IntFeasTol': 1e-3 if approx else 1e-5,
'Presolve': 2,
'StartNodeLimit': 10000000
}
ilpsolver = ILPSolver(g,
budget,
gurobi_params=param_dict,
seed_s=seed_s,
eps_noise=eps_noise,
imposed_schedule=imposed_schedule,
solve_r=solve_r,
write_model_file=write_model_file)
ilpsolver.build_model()
try:
r, s, u, free_e = ilpsolver.solve()
ilp_feasible = True
except ValueError as e:
logging.exception(e)
r, s, u, free_e = (None, None, None, None)
ilp_feasible = False
ilp_aux_data = ILPAuxData(U=u,
Free_E=free_e,
ilp_approx=approx,
ilp_time_limit=time_limit,
ilp_eps_noise=eps_noise,
ilp_num_constraints=ilpsolver.m.numConstrs,
ilp_num_variables=ilpsolver.m.numVars,
ilp_imposed_schedule=imposed_schedule)
schedule, aux_data = schedule_from_rs(g, r, s)
return ScheduledResult(
solve_strategy=SolveStrategy.OPTIMAL_ILP_GC,
solver_budget=budget,
feasible=ilp_feasible,
schedule=schedule,
schedule_aux_data=aux_data,
solve_time_s=ilpsolver.solve_time,
ilp_aux_data=ilp_aux_data,
)
def write_result(self, key: RedisCacheKey, result: ScheduledResult, ilp_time_limit: int = -1):
return self.redis_conn.set(key.key(ilp_time_limit), result.dumps())
def lower_bound_lp_relaxation(
g: DFGraph,
budget: int,
seed_s: Optional[np.ndarray] = None,
approx=True,
imposed_schedule=ImposedSchedule.FULL_SCHEDULE,
time_limit: Optional[int] = None,
write_log_file: Optional[PathLike] = None,
print_to_console=True,
write_model_file: Optional[PathLike] = None,
eps_noise=0.01,
solver_cores=os.cpu_count(),
):
param_dict = {
"LogToConsole": 1 if print_to_console else 0,
"LogFile": str(write_log_file) if write_log_file is not None else "",
"Threads": solver_cores,
"TimeLimit": math.inf if time_limit is None else time_limit,
"OptimalityTol": 1e-2 if approx else 1e-4,
"IntFeasTol": 1e-3 if approx else 1e-5,
"Presolve": 2,
"StartNodeLimit": 10000000,
}
lpsolver = ILPSolver(
g,
budget,
gurobi_params=param_dict,
seed_s=seed_s,
integral=False,
solve_r=False,
eps_noise=eps_noise,
imposed_schedule=imposed_schedule,
write_model_file=write_model_file,
)
lpsolver.build_model()
try:
r, s, u, free_e = lpsolver.solve()
lp_feasible = True
except ValueError as e:
logging.exception(e)
r, s, u, free_e = (None, None, None, None)
lp_feasible = False
total_ram = u.max()
total_cpu = lpsolver.m.getObjective().getValue()
aux_data = SchedulerAuxData(R=r,
S=s,
cpu=total_cpu,
peak_ram=total_ram,
activation_ram=total_ram,
mem_timeline=None,
mem_grid=None)
return ScheduledResult(
solve_strategy=SolveStrategy.LB_LP,
solver_budget=budget,
feasible=lp_feasible,
schedule=None,
schedule_aux_data=aux_data,
solve_time_s=lpsolver.solve_time,
ilp_aux_data=ILPAuxData(
U=u,
Free_E=free_e,
ilp_approx=approx,
ilp_time_limit=time_limit,
ilp_eps_noise=eps_noise,
ilp_num_constraints=lpsolver.m.numConstrs,
ilp_num_variables=lpsolver.m.numVars,
approx_deterministic_round_threshold=None,
),
)