marker=marker, s=markersize**2) ax.hlines(y=y, xmin=xlim_min, xmax=x, linestyles="dashed", color=color) ax.hlines(y=y, xmin=x, xmax=xlim_max, color=color, zorder=2) legend_elements.append( Line2D([0], [0], lw=2, label=label, color=color, marker=marker, markersize=markersize)) ax.set_xlim([xlim_min, xlim_max]) # Plot platform memory ylim_min, ylim_max = ax.get_ylim() mem_gb = platform_memory(args.platform) / 1e9 if xlim_min <= mem_gb <= xlim_max: ax.vlines(x=mem_gb, ymin=ylim_min, ymax=ylim_max, linestyles="dotted", color="b") legend_elements.append( Line2D([0], [0], lw=2, label="{} memory".format(pretty_platform_name( args.platform)), color="b", linestyle="dotted")) ax.set_ylim([ylim_min, ylim_max])
def get_solutions_to_evaluate( solve_strategy: SolveStrategy, model_name: str, batch_size: int, platform: str, input_shape=None, model_version="v1", buffer_mem: int = 0) -> List[Tuple[RSResult, str]]: """ :param solve_strategy: :param model_name: :param batch_size: :param platform: :param input_shape: :param model_version: :return: Instance of RSResult, or None. Returns None if the solution is not available in cache or no solution is available under the budget """ logger = setup_logger("test_execution_get_solution") # Load all results for this configuration, regardless of budget key_prefix = RedisCache.make_key(platform=platform, model_name=model_name, model_version=model_version, batch_size=batch_size, input_shape=input_shape) cache = RedisCache(key_prefix=key_prefix) cost_file = f"b{batch_size}_{platform}.npy" logger.info( f"Querying results for SS={solve_strategy}, model_name=f{model_name}, bs=f{batch_size}, " f"platform={platform}, cost_file={cost_file}, key prefix={key_prefix}") results, keys = cache.read_results(solver=solve_strategy, cost_file=cost_file, model_name=model_name) if not results: logger.error( f"No solutions found in cache for SS={solve_strategy}, model_name=f{model_name}, " f"bs=f{batch_size}, platform={platform}, cost_file={cost_file}, key prefix={key_prefix}" ) return [] # Filter results to those that abide by the budget platform_budget = platform_memory(platform) within_budget = [] for result, key in zip(results, keys): if not result.peak_ram: logger.warn(f"Falsey peak ram? {result.peak_ram}") continue if result.peak_ram + buffer_mem <= platform_budget: within_budget.append((result, key)) logger.info( f"Out of {len(results)} solver results, {len(within_budget)} had <= {platform_budget} - {buffer_mem} peak ram" ) if not within_budget: logger.warn( f"While {len(results)} solutions were found in cache, no solutions are within budget" ) return [] # Return solution in increasing order of cost within_budget.sort(key=lambda r: r[0].cpu) return within_budget # Return min compute solution min_compute = within_budget[0] for result in within_budget: if result[0].cpu < min_compute[0].cpu: min_compute = result logger.info( f"Using solution with f{min_compute[0].cpu} compute, f{min_compute[0].peak_ram} ram" ) return min_compute
else: cost_model = CostModel(model_name, args.platform, log_base, quantization=5) cost_model.fit() cost_model.plot_costs() model = get_keras_model(model_name, input_shape=args.input_shape) tf.keras.utils.plot_model(model, to_file=log_base / "plot_{}.png".format(model_name), show_shapes=True, show_layer_names=True) platform_ram = platform_memory("p32xlarge") bs_futures = defaultdict(list) # type: Dict[int, List] bs_param_ram_cost = {} # type: Dict[int, int] bs_fwd2xcost = {} # type: Dict[int, int] rg = list( range(args.batch_size_min, args.batch_size_max, args.batch_size_increment)) for bs in tqdm(rg, desc="Event dispatch"): while not ray.is_initialized(): ray.init( temp_dir="/tmp/ray_checkpoint_" + str(str(uuid.uuid4())[:8]), redis_password=str(uuid.uuid1()), num_cpus=os.cpu_count() - 2, ) futures = []
else: cost_model = CostModel(model_name, args.platform, log_base, quantization=5) cost_model.fit() cost_model.plot_costs() model = get_keras_model(model_name, input_shape=args.input_shape) tf.keras.utils.plot_model(model, to_file=log_base / "plot_{}.png".format(model_name), show_shapes=True, show_layer_names=True) platform_ram = platform_memory("p32xlarge") bs_futures = defaultdict(list) # type: Dict[int, List] bs_fwd2xcost = {} # type: Dict[int, int] # load model at batch size g = dfgraph_from_keras(model, batch_size=1, cost_model=cost_model, loss_cpu_cost=0, loss_ram_cost=(4)) plot_dfgraph(g, log_base, name=model_name) model_file = str(log_base / "max_bs_{}.mps".format(model_name)) param_dict = { "LogToConsole": 1, "LogFile": str(log_base / "max_bs_{}.solve.log".format(model_name)), "Threads": os.cpu_count(),
return data * 1e-9 if __name__ == "__main__": args = extract_params() if args.model_name == "linear16": N = 16 B = 8 scratch_dir = checkmate_data_dir( ) / "approxcomparison" / args.model_name / f"budget{B}" scratch_dir.mkdir(parents=True, exist_ok=True) g = gen_linear_graph(N) else: B = platform_memory(args.platform) scratch_dir = checkmate_data_dir( ) / "approxcomparison" / args.model_name / f"budget{B}" scratch_dir.mkdir(parents=True, exist_ok=True) # load costs, and plot optionally, if platform is not flops print("Loading costs") if args.platform == "flops": cost_model = None else: cost_model = CostModel(args.model_name, args.platform, scratch_dir, quantization=5) cost_model.fit()