def optimize(self): cs = ConfigurationSpace() if 'optimizer' in self.params: optimizer = CategoricalHyperparameter('optimizer', self.params['optimizer']) cs.add_hyperparameter(optimizer) scenario = Scenario({ "run_obj": "quality", "runcount-limit": 5, "cutoff-time": 10, "cs": cs, "deterministic": "true" }) print(cs.get_default_configuration()) def_value = self.svm_from_config(cs.get_default_configuration()) smac = SMAC4HPO(scenario=scenario, rng=np.random.RandomState(42), tae_runner=self.svm_from_config) incumbent = smac.optimize() __params = incumbent.get_dictionary() inc_value = self.svm_from_config(incumbent) print(__params) model = create_model(**__params) hist = model.fit(self.x_train, self.y_train, batch_size=128, epochs=6) time = smac.stats.wallclock_time_used loss, accuracy, f1_score, precision, recall = model.evaluate( self.x_test, self.y_test, verbose=0) del smac self.result = { 'accuracy': accuracy, 'time': time, 'best_params': __params }
def optimize(): # We load the iris-dataset (a widely used benchmark) iris = datasets.load_iris() #logger = logging.getLogger("SVMExample") logging.basicConfig(level=logging.INFO) # logging.DEBUG for debug output # Build Configuration Space which defines all parameters and their ranges cs = ConfigurationSpace() # We define a few possible types of SVM-kernels and add them as "kernel" to our cs kernel = CategoricalHyperparameter("kernel", ["linear", "rbf", "poly", "sigmoid"], default="poly") cs.add_hyperparameter(kernel) # There are some hyperparameters shared by all kernels C = UniformFloatHyperparameter("C", 0.001, 1000.0, default=1.0) shrinking = CategoricalHyperparameter("shrinking", ["true", "false"], default="true") cs.add_hyperparameters([C, shrinking]) # Others are kernel-specific, so we can add conditions to limit the searchspace degree = UniformIntegerHyperparameter("degree", 1, 5, default=3) # Only used by kernel poly coef0 = UniformFloatHyperparameter("coef0", 0.0, 10.0, default=0.0) # poly, sigmoid cs.add_hyperparameters([degree, coef0]) use_degree = InCondition(child=degree, parent=kernel, values=["poly"]) use_coef0 = InCondition(child=coef0, parent=kernel, values=["poly", "sigmoid"]) cs.add_conditions([use_degree, use_coef0]) # This also works for parameters that are a mix of categorical and values from a range of numbers # For example, gamma can be either "auto" or a fixed float gamma = CategoricalHyperparameter("gamma", ["auto", "value"], default="auto") # only rbf, poly, sigmoid gamma_value = UniformFloatHyperparameter("gamma_value", 0.0001, 8, default=1) cs.add_hyperparameters([gamma, gamma_value]) # We only activate gamma_value if gamma is set to "value" cs.add_condition(InCondition(child=gamma_value, parent=gamma, values=["value"])) # And again we can restrict the use of gamma in general to the choice of the kernel cs.add_condition(InCondition(child=gamma, parent=kernel, values=["rbf", "poly", "sigmoid"])) # Scenario object scenario = Scenario("test/test_files/svm_scenario.txt") # Example call of the function # It returns: Status, Cost, Runtime, Additional Infos def_value = svm_from_cfg(cs.get_default_configuration()) print("Default Value: %.2f" % (def_value)) # Optimize, using a SMAC-object print("Optimizing! Depending on your machine, this might take a few minutes.") smac = SMAC(scenario=scenario, rng=np.random.RandomState(42), tae_runner=svm_from_cfg) incumbent = smac.optimize() inc_value = svm_from_cfg(incumbent) print("Optimized Value: %.2f" % (inc_value))
def main_loop(problem): logging.basicConfig(level=logging.INFO) # logging.DEBUG for debug output cs = ConfigurationSpace() n_neighbors = UniformIntegerHyperparameter("n_neighbors", 2,10, default_value=5) cs.add_hyperparameter(n_neighbors) weights = CategoricalHyperparameter("weights", ["uniform","distance"], default_value="uniform") algorithm = CategoricalHyperparameter("algorithm", ["ball_tree", "kd_tree","brute","auto"], default_value="auto") cs.add_hyperparameters([weights, algorithm]) leaf_size = UniformIntegerHyperparameter("leaf_size", 1, 100, default_value=50) cs.add_hyperparameter(leaf_size) use_leaf_size= InCondition(child=leaf_size, parent=algorithm, values=["ball_tree","kd_tree"]) cs.add_condition(use_leaf_size) p = UniformIntegerHyperparameter("p", 1,3, default_value=2) cs.add_hyperparameter(p) # Scenario object max_eval=100000 scenario = Scenario({"run_obj": "quality", # we optimize quality (alternatively runtime) "runcount-limit": max_eval, # maximum function evaluations "cs": cs, # configuration space "shared_model": True, "output_dir": "/home/naamah/Documents/CatES/result_All/smac/KNN/run_{}_{}_{}".format(max_eval, datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d_%H:%M:%S'), problem) # "output_dir": "/home/naamah/Documents/CatES/result_All/smac/KNN/{}/run_{}_{}".format(problem,max_eval, datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d_%H:%M:%S_%f')), # "input_psmac_dirs":"/home/naamah/Documents/CatES/result_All/", # "deterministic": "true" }) # Example call of the function # It returns: Status, Cost, Runtime, Additional Infos def_value = svm_from_cfg(cs.get_default_configuration()) print("Default Value: %.2f" % (def_value)) # Optimize, using a SMAC-object print("Optimizing! Depending on your machine, this might take a few minutes.") smac = SMAC(scenario=scenario,tae_runner=svm_from_cfg) incumbent = smac.optimize() inc_value = svm_from_cfg(incumbent) print("Optimized Value: %.2f" % (inc_value)) return (incumbent) # main_loop()
def test_illegal_input(self): """ Testing illegal input in smbo """ cs = ConfigurationSpace() cs.add_hyperparameter(UniformFloatHyperparameter('test', 1, 10, 5)) scen = Scenario({'run_obj': 'quality', 'cs': cs}) stats = Stats(scen) # Recorded runs but no incumbent. stats.ta_runs = 10 smac = SMAC(scen, stats=stats, rng=np.random.RandomState(42)) self.assertRaises(ValueError, smac.optimize) # Incumbent but no recoreded runs. incumbent = cs.get_default_configuration() smac = SMAC(scen, restore_incumbent=incumbent, rng=np.random.RandomState(42)) self.assertRaises(ValueError, smac.optimize)
scenario = Scenario({ "run_obj": "quality", # we optimize quality (alternative runtime) "runcount-limit": 200, # at most 200 function evaluations "cs": cs, # configuration space "deterministic": "true", #"tuner-timeout" : 1, #"wallclock_limit": 2 }) stats = Stats(scenario) # register function to be optimize taf = ExecuteTAFunc(rosenbrock_4d, stats) # example call of the function # it returns: Status, Cost, Runtime, Additional Infos def_value = taf.run(cs.get_default_configuration())[1] print("Default Value: %.2f" % (def_value)) # Optimize smbo = SMBO(scenario=scenario, tae_runner=taf, stats=stats, rng=np.random.RandomState(42)) try: smbo.run(max_iters=999) finally: smbo.stats.print_stats() print("Final Incumbent: %s" % (smbo.incumbent)) inc_value = taf.run(smbo.incumbent)[1] print("Optimized Value: %.2f" % (inc_value))
def main_loop(problem): logging.basicConfig(level=logging.INFO) # logging.DEBUG for debug output cs = ConfigurationSpace() n_estimators = UniformIntegerHyperparameter("n_estimators", 5, 50, default_value=10) #criterion = CategoricalHyperparameter("criterion", ["mse", "mae"], default_value="mse") min_samples_split = UniformIntegerHyperparameter("min_samples_split", 2, 20, default_value=2) min_samples_leaf = UniformIntegerHyperparameter("min_samples_leaf", 1, 20, default_value=1) min_weight_fraction_leaf = UniformFloatHyperparameter( "min_weight_fraction_leaf", 0.0, 0.5, default_value=0.0) max_leaf_nodes = UniformIntegerHyperparameter("max_leaf_nodes", 10, 1000, default_value=100) min_impurity_decrease = UniformFloatHyperparameter("min_impurity_decrease", 0.0, 0.5, default_value=0.0) warm_start = CategoricalHyperparameter("warm_start", ["true", "false"], default_value="false") cs.add_hyperparameters([ n_estimators, min_weight_fraction_leaf, min_samples_split, min_samples_leaf, max_leaf_nodes, warm_start, min_impurity_decrease ]) max_features = CategoricalHyperparameter( "max_features", ["auto", "log2", "sqrt", "int", "None", "float"], default_value="auto") # only rbf, poly, sigmoid max_features_int = UniformIntegerHyperparameter("max_features_int", 2, len(X[0]), default_value=5) max_features_float = UniformFloatHyperparameter("max_features_float", 0.0, 0.9, default_value=0.0) cs.add_hyperparameters( [max_features, max_features_int, max_features_float]) use_max_features_int = InCondition(child=max_features_int, parent=max_features, values=["int"]) use_max_features_float = InCondition(child=max_features_float, parent=max_features, values=["float"]) cs.add_conditions([use_max_features_int, use_max_features_float]) max_depth = CategoricalHyperparameter("max_depth", ["None", "value"], default_value="None") max_depth_value = UniformIntegerHyperparameter("max_depth_value", 2, 20, default_value=5) cs.add_hyperparameters([max_depth, max_depth_value]) cs.add_condition( InCondition(child=max_depth_value, parent=max_depth, values=["value"])) random_state = CategoricalHyperparameter("random_state", ["None", "value"], default_value="None") random_state_value = UniformIntegerHyperparameter("random_state_value", 1, 20, default_value=1) cs.add_hyperparameters([random_state, random_state_value]) cs.add_condition( InCondition(child=random_state_value, parent=random_state, values=["value"])) with open("/home/naamah/Documents/CatES/result_All/X1.p", "rb") as fp: X = pickle.load(fp) # Scenario object max_eval = 100000 scenario = Scenario({ "run_obj": "quality", # we optimize quality (alternatively runtime) "runcount-limit": max_eval, # maximum function evaluations "cs": cs, # configuration space "shared_model": True, "output_dir": "/home/naamah/Documents/CatES/result_All/smac/RF/run_{}_{}_{}".format( max_eval, datetime.datetime.fromtimestamp( time.time()).strftime('%Y-%m-%d_%H:%M:%S'), problem), "input_psmac_dirs": "/home/naamah/Documents/CatES/result_All/smac/psmac", "deterministic": "False" }) def_value = svm_from_cfg(cs.get_default_configuration()) print("Default Value: %.2f" % (def_value)) # Optimize, using a SMAC-object print( "Optimizing! Depending on your machine, this might take a few minutes." ) smac = SMAC(scenario=scenario, tae_runner=svm_from_cfg) incumbent = smac.optimize() inc_value = svm_from_cfg(incumbent) print("Optimized Value: %.2f" % (inc_value)) return (incumbent) # main_loop()
# Scenario object scenario = Scenario({ "run_obj": "quality", # we optimize quality # (alternatively runtime) "runcount-limit": 10, "cs": cs, # configuration space "deterministic": "false", "shared_model": True, "input_psmac_dirs": "smac-output-learch", "cutoff_time": 9000, "wallclock_limit": 'inf' }) # Example call of the function # It returns: Status, Cost, Runtime, Additional Infos def_value = learch(cs.get_default_configuration()) print("Default Value: %.2f" % def_value) # Optimize, using a SMAC-object print("Optimizing! Depending on your machine, this might take a few minutes.") smac = SMAC4HPO(scenario=scenario, rng=np.random.RandomState(42), tae_runner=learch) # Start optimization try: incumbent = smac.optimize() finally: incumbent = smac.solver.incumbent
default_value=4) mutation_prob = UniformFloatHyperparameter("mutation_prob", 0.1, 0.9, default_value=0.5) cs.add_hyperparameters( [num_generations, number_chromosomes, num_parents_mating, mutation_prob]) # Scenario object scenario = Scenario({ "run_obj": "quality", # we optimize quality (alternatively runtime) "runcount-limit": 10, # max. number of function evaluations; for this example set to a low number "cs": cs, # configuration space "deterministic": "true" }) # Example call of the function # It returns: Status, Cost, Runtime, Additional Infos def_value = gawd(cs.get_default_configuration()) # Optimize, using a SMAC-object print("Optimizing! Depending on your machine, this might take a few minutes.") smac = SMAC4BO( scenario=scenario, rng=np.random.RandomState(42), tae_runner=gawd, ) smac.optimize()
"eta": 3 } # To optimize, we pass the function to the SMAC-object smac = SMAC4MF( scenario=scenario, rng=np.random.RandomState(42), tae_runner=mlp_from_cfg, intensifier_kwargs=intensifier_kwargs, ) tae = smac.get_tae_runner() # Example call of the function with default values # It returns: Status, Cost, Runtime, Additional Infos def_value = tae.run(config=cs.get_default_configuration(), budget=max_epochs, seed=0)[1] print("Value for default configuration: %.4f" % def_value) # Start optimization try: incumbent = smac.optimize() finally: incumbent = smac.solver.incumbent inc_value = tae.run(config=incumbent, budget=max_epochs, seed=0)[1] print("Optimized Value: %.4f" % inc_value)
scenario = Scenario({"run_obj": "quality", # we optimize quality (alternative runtime) "runcount-limit": 50, # maximum number of function evaluations "cs": cs, # configuration space "deterministic": "true", "memory_limit": 3072, # adapt this to reasonable value for your hardware "shared_model": True, "input_psmac_dirs": "smac3-output*" }) # To optimize, we pass the function to the SMAC-object smac = SMAC(scenario=scenario, rng=np.random.RandomState(42), tae_runner=rf_from_cfg) # Example call of the function with default values # It returns: Status, Cost, Runtime, Additional Infos def_value = smac.get_tae_runner().run(cs.get_default_configuration(), 1)[1] print("Value for default configuration: %.2f" % (def_value)) # Start optimization try: incumbent = smac.optimize() finally: incumbent = smac.solver.incumbent incumbent._values mlflow.log_param("city",city) mlflow.log_param("model","rf") all_feats = "" for elem in features: if all_feats == "":
# Scenario object scenario = Scenario({ "run_obj": "quality", # we optimize quality # (alternatively runtime) "runcount-limit": 10, "cs": cs, # configuration space "deterministic": "false", "shared_model": True, "input_psmac_dirs": "smac-output-maxEnt", "cutoff_time": 9000, "wallclock_limit": 'inf' }) # Example call of the function # It returns: Status, Cost, Runtime, Additional Infos def_value = maxEnt(cs.get_default_configuration()) print("Default Value: %.2f" % def_value) # Optimize, using a SMAC-object print("Optimizing! Depending on your machine, this might take a few minutes.") smac = SMAC4HPO(scenario=scenario, rng=np.random.RandomState(42), tae_runner=maxEnt) # Start optimization try: incumbent = smac.optimize() finally: incumbent = smac.solver.incumbent
for i in range(total_features): param_id = UniformIntegerHyperparameter( str(i), 0, 1, default_value=0) # Only used by kernel poly cs.add_hyperparameter(param_id) # Scenario object scenario = Scenario({ "run_obj": "quality", # we optimize quality (alternatively runtime) "runcount-limit": 200, # maximum function evaluations "cs": cs, # configuration space "deterministic": "true" }) # Example call of the function # It returns: Status, Cost, Runtime, Additional Infos def_value = simulator_fun(cs.get_default_configuration()) print("Default Value: %.2f" % (def_value)) # Optimize, using a SMAC-object print("Optimizing! Depending on your machine, this might take a few minutes.") total_value = np.zeros(200) for i in range(5): smac = SMAC(scenario=scenario, rng=np.random.RandomState(42), tae_runner=simulator_fun) incumbent = smac.optimize() print(smac.trajectory) idx = 0 for j in range(200): if (idx < len(smac.trajectory)):
1.0, default_value=0.1, log=True) max_features = UniformIntegerHyperparameter("max_features", 2, 10, default_value=4) cs.add_hyperparameters([min_samples_split, max_features]) subsample = UniformFloatHyperparameter("subsample", 0.5, 1, default_value=0.8) cs.add_hyperparameter(subsample) cfg = cs.get_default_configuration() clf = GradientBoostingClassifier(**cfg, random_state=0).fit(X_train, y_train) def_test_score = 1 - clf.score(X_test, y_test) print("Default cross validation score: %.2f" % (xgboost_from_cfg(cfg))) print("Default test score: %.2f" % def_test_score) # scenario object scenario = Scenario({ "run_obj": "quality", "runcount-limit": 100, "cs": cs, # the evaluations are not deterministic, we need to repeat each # configuration several times and take the mean value of these repetitions "deterministic": "false",
cs = ConfigurationSpace() max_depth = UniformIntegerHyperparameter("max_depth", 1, 10, default_value=3) cs.add_hyperparameter(max_depth) learning_rate = UniformFloatHyperparameter("learning_rate", 0.01, 1.0, default_value=1.0, log=True) cs.add_hyperparameter(learning_rate) min_samples_split = UniformFloatHyperparameter("min_samples_split", 0.01, 1.0, default_value=0.1, log=True) max_features = UniformIntegerHyperparameter("max_features", 2, 10, default_value=4) cs.add_hyperparameters([min_samples_split, max_features]) subsample = UniformFloatHyperparameter("subsample", 0.5, 1, default_value=0.8) cs.add_hyperparameter(subsample) print("Default cross validation score: %.2f" % (xgboost_from_cfg(cs.get_default_configuration()))) cfg = cs.get_default_configuration() clf = GradientBoostingClassifier(**cfg, random_state=0).fit(X_train, y_train) def_test_score = 1 - clf.score(X_test, y_test) print("Default test score: %.2f" % def_test_score) # scenario object scenario = Scenario({ "run_obj": "quality", "runcount-limit": 100, "cs": cs, # the evaluations are not deterministic, we need to repeat each # configuration several times and take the mean value of these repetitions "deterministic": "false", "wallclock_limit": 120, "maxR": 3, # Each configuration will be evaluated maximal 3 times with various seeds
cs.add_hyperparameters([learning_rate, step_size_scalar, l2_regularizer, N]) # Scenario object scenario = Scenario({"run_obj": "quality", # we optimize quality # (alternatively runtime) "runcount-limit": 2, "cs": cs, # configuration space "deterministic": "false", "shared_model": True, "input_psmac_dirs": "smac-output-learch", "cutoff_time": 9000, "wallclock_limit": 'inf' }) # Example call of the function # It returns: Status, Cost, Runtime, Additional Infos def_value = learch_variant(cs.get_default_configuration()) print("Default Value: %.2f" % def_value) # Optimize, using a SMAC-object print("Optimizing! Depending on your machine, this might take a few minutes.") smac = SMAC4HPO(scenario=scenario, rng=np.random.RandomState(42), tae_runner=learch_variant) # Start optimization try: incumbent = smac.optimize() finally: incumbent = smac.solver.incumbent
"cs": cs, # configuration space "deterministic": False, "shared_model": True, "output_dir": "smac3-output/", "input_psmac_dirs": "smac3-output/run_*", "instances": INSTANCES }) i = int(sys.argv[1]) # Optimize, using a SMAC-object print("Optimizing! Depending on your machine, this might take a few minutes.") smac = BOHB4HPO(scenario=scenario, rng=np.random.RandomState(i), tae_runner=gadma_from_cfg, run_id=i, intensifier_kwargs=intensifier_kwargs) incumbent = smac.optimize() def_costs = [] for i in INSTANCES: cost = smac.get_tae_runner().run(cs.get_default_configuration(), i[0])[1] def_costs.append(cost) print("Value for default configuration: %.4f" % (np.mean(def_costs))) inc_costs = [] for i in INSTANCES: cost = smac.get_tae_runner().run(incumbent, i[0])[1] inc_costs.append(cost) print("Optimized Value: %.4f" % (np.mean(inc_costs)))
def main_loop(problem): #logger = logging.getLogger("SVMExample") logging.basicConfig(level=logging.INFO) # logging.DEBUG for debug output cs = ConfigurationSpace() kernel = CategoricalHyperparameter("kernel", ["rbf", "sigmoid"], default_value="rbf") cs.add_hyperparameter(kernel) C = UniformFloatHyperparameter("C", 1, 1000.0, default_value=900.0) shrinking = CategoricalHyperparameter("shrinking", ["true", "false"], default_value="false") cs.add_hyperparameters([C, shrinking]) #degree = UniformIntegerHyperparameter("degree", 1, 3 ,default_value=3) # Only used by kernel poly coef0 = UniformFloatHyperparameter("coef0", 0.0, 1.0, default_value=0.0) # poly, sigmoid #cs.add_hyperparameters([degree, coef0]) cs.add_hyperparameter(coef0) #use_degree = InCondition(child=degree, parent=kernel, values=["poly"]) use_coef0 = InCondition(child=coef0, parent=kernel, values=["sigmoid"]) #cs.add_conditions([use_degree, use_coef0]) cs.add_condition(use_coef0) gamma = CategoricalHyperparameter( "gamma", ["auto", "value"], default_value="auto") # only rbf, poly, sigmoid gamma_value = UniformFloatHyperparameter("gamma_value", 0.001, 8, default_value=1) cs.add_hyperparameters([gamma, gamma_value]) cs.add_condition( InCondition(child=gamma_value, parent=gamma, values=["value"])) cs.add_condition( InCondition(child=gamma, parent=kernel, values=["rbf", "sigmoid"])) epsilon = UniformFloatHyperparameter("epsilon", 0.001, 5.0, default_value=0.1) cs.add_hyperparameter(epsilon) # Scenario object max_eval = 100000 scenario = Scenario({ "run_obj": "quality", # we optimize quality (alternatively runtime) "runcount-limit": max_eval, # maximum function evaluations "cs": cs, # configuration space "shared_model": True, "output_dir": "/home/naamah/Documents/CatES/result_All/smac/svm/run_{}_{}_{}".format( max_eval, datetime.datetime.fromtimestamp( time.time()).strftime('%Y-%m-%d_%H:%M:%S'), problem), # "input_psmac_dirs": "/home/naamah/Documents/CatES/result_All/smac/svm/{}/run_{}_{}/*".format(problem,max_eval, datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d_%H:%M:%S_%f')), "deterministic": "False" #"instance_file":"/home/naamah/Documents/CatES/result_All", #"test_instance_file":"/home/naamah/Documents/CatES/result_All" }) # Example call of the function # It returns: Status, Cost, Runtime, Additional Infos def_value = svm_from_cfg(cs.get_default_configuration()) print("Default Value: %.2f" % (def_value)) # Optimize, using a SMAC-object print( "Optimizing! Depending on your machine, this might take a few minutes." ) smac = SMAC(scenario=scenario, tae_runner=svm_from_cfg) incumbent = smac.optimize() inc_value = svm_from_cfg(incumbent) print("Optimized Value: %.2f" % (inc_value)) return (incumbent)
def quant_post_hpo( executor, place, model_dir, quantize_model_path, train_sample_generator=None, eval_sample_generator=None, train_dataloader=None, eval_dataloader=None, eval_function=None, model_filename=None, params_filename=None, save_model_filename='__model__', save_params_filename='__params__', scope=None, quantizable_op_type=["conv2d", "depthwise_conv2d", "mul"], is_full_quantize=False, weight_bits=8, activation_bits=8, weight_quantize_type=['channel_wise_abs_max'], algo=["KL", "hist", "avg", "mse"], bias_correct=[True, False], hist_percent=[0.98, 0.999], ### uniform sample in list. batch_size=[10, 30], ### uniform sample in list. batch_num=[10, 30], ### uniform sample in list. optimize_model=False, is_use_cache_file=False, cache_dir="./temp_post_training", runcount_limit=30): """ The function utilizes static post training quantization method to quantize the fp32 model. It uses calibrate data to calculate the scale factor of quantized variables, and inserts fake quantization and dequantization operators to obtain the quantized model. Args: executor(paddle.static.Executor): The executor to load, run and save the quantized model. place(paddle.CPUPlace or paddle.CUDAPlace): This parameter represents the executor run on which device. model_dir(str): The path of fp32 model that will be quantized, and the model and params that saved by ``paddle.static.io.save_inference_model`` are under the path. quantize_model_path(str): The path to save quantized model using api ``paddle.static.io.save_inference_model``. train_sample_generator(Python Generator): The sample generator provides calibrate data for DataLoader, and it only returns a sample every time. eval_sample_generator(Python Generator): The sample generator provides evalution data for DataLoader, and it only returns a sample every time. model_filename(str, optional): The name of model file. If parameters are saved in separate files, set it as 'None'. Default: 'None'. params_filename(str, optional): The name of params file. When all parameters are saved in a single file, set it as filename. If parameters are saved in separate files, set it as 'None'. Default : 'None'. save_model_filename(str): The name of model file to save the quantized inference program. Default: '__model__'. save_params_filename(str): The name of file to save all related parameters. If it is set None, parameters will be saved in separate files. Default: '__params__'. scope(paddle.static.Scope, optional): The scope to run program, use it to load and save variables. If scope is None, will use paddle.static.global_scope(). quantizable_op_type(list[str], optional): The list of op types that will be quantized. Default: ["conv2d", "depthwise_conv2d", "mul"]. is_full_quantize(bool): if True, apply quantization to all supported quantizable op type. If False, only apply quantization to the input quantizable_op_type. Default is False. weight_bits(int, optional): quantization bit number for weights. activation_bits(int): quantization bit number for activation. weight_quantize_type(str): quantization type for weights, support 'abs_max' and 'channel_wise_abs_max'. Compared to 'abs_max', the model accuracy is usually higher when using 'channel_wise_abs_max'. optimize_model(bool, optional): If set optimize_model as True, it applies some passes to optimize the model before quantization. So far, the place of executor must be cpu it supports fusing batch_norm into convs. is_use_cache_file(bool): This param is deprecated. cache_dir(str): This param is deprecated. runcount_limit(int): max. number of model quantization. Returns: None """ global g_quant_config g_quant_config = QuantConfig( executor, place, model_dir, quantize_model_path, algo, hist_percent, bias_correct, batch_size, batch_num, train_sample_generator, eval_sample_generator, train_dataloader, eval_dataloader, eval_function, model_filename, params_filename, save_model_filename, save_params_filename, scope, quantizable_op_type, is_full_quantize, weight_bits, activation_bits, weight_quantize_type, optimize_model, is_use_cache_file, cache_dir) cs = ConfigurationSpace() hyper_params = [] if 'hist' in algo: hist_percent = UniformFloatHyperparameter( "hist_percent", hist_percent[0], hist_percent[1], default_value=hist_percent[0]) hyper_params.append(hist_percent) if len(algo) > 1: algo = CategoricalHyperparameter("algo", algo, default_value=algo[0]) hyper_params.append(algo) else: algo = algo[0] if len(bias_correct) > 1: bias_correct = CategoricalHyperparameter("bias_correct", bias_correct, default_value=bias_correct[0]) hyper_params.append(bias_correct) else: bias_correct = bias_correct[0] if len(weight_quantize_type) > 1: weight_quantize_type = CategoricalHyperparameter("weight_quantize_type", \ weight_quantize_type, default_value=weight_quantize_type[0]) hyper_params.append(weight_quantize_type) else: weight_quantize_type = weight_quantize_type[0] if len(batch_size) > 1: batch_size = UniformIntegerHyperparameter("batch_size", batch_size[0], batch_size[1], default_value=batch_size[0]) hyper_params.append(batch_size) else: batch_size = batch_size[0] if len(batch_num) > 1: batch_num = UniformIntegerHyperparameter("batch_num", batch_num[0], batch_num[1], default_value=batch_num[0]) hyper_params.append(batch_num) else: batch_num = batch_num[0] if len(hyper_params) == 0: quant_post( \ executor=g_quant_config.executor, \ scope=g_quant_config.scope, \ model_dir=g_quant_config.float_infer_model_path, \ quantize_model_path=g_quant_model_cache_path, \ sample_generator=g_quant_config.train_sample_generator, \ data_loader=g_quant_config.train_dataloader, model_filename=g_quant_config.model_filename, \ params_filename=g_quant_config.params_filename, \ save_model_filename=g_quant_config.save_model_filename, \ save_params_filename=g_quant_config.save_params_filename, \ quantizable_op_type=g_quant_config.quantizable_op_type, \ activation_quantize_type="moving_average_abs_max", \ weight_quantize_type=weight_quantize_type, \ algo=algo, \ hist_percent=hist_percent, \ bias_correction=bias_correct, \ batch_size=batch_size, \ batch_nums=batch_num) return cs.add_hyperparameters(hyper_params) scenario = Scenario({ "run_obj": "quality", # we optimize quality (alternative runtime) "runcount-limit": runcount_limit, # max. number of function evaluations; for this example set to a low number "cs": cs, # configuration space "deterministic": "True", "limit_resources": "False", "memory_limit": 4096 # adapt this to reasonable value for your hardware }) # To optimize, we pass the function to the SMAC-object smac = SMAC4HPO(scenario=scenario, rng=np.random.RandomState(42), tae_runner=quantize) # Example call of the function with default values # It returns: Status, Cost, Runtime, Additional Infos def_value = smac.get_tae_runner().run(cs.get_default_configuration(), 1)[1] print("Value for default configuration: %.8f" % def_value) # Start optimization try: incumbent = smac.optimize() finally: incumbent = smac.solver.incumbent inc_value = smac.get_tae_runner().run(incumbent, 1)[1] print("Optimized Value: %.8f" % inc_value) print("quantize completed")
'instance_order': 'shuffle_once' } # To optimize, we pass the function to the SMAC-object smac = ROAR(scenario=scenario, rng=np.random.RandomState(42), tae_runner=mlp_from_cfg, intensifier=SuccessiveHalving, intensifier_kwargs=intensifier_kwargs, initial_design=RandomConfigurations, n_jobs=4) # Example call of the function with default values # It returns: Status, Cost, Runtime, Additional Infos def_value = smac.get_tae_runner().run( config=cs.get_default_configuration(), instance='1', budget=25, seed=0)[1] print("Value for default configuration: %.4f" % def_value) # Start optimization try: incumbent = smac.optimize() finally: incumbent = smac.solver.incumbent inc_value = smac.get_tae_runner().run(config=incumbent, instance='1', budget=25, seed=0)[1] print("Optimized Value: %.4f" % inc_value)
intensifier_kwargs = { 'initial_budget': 5, 'max_budget': max_epochs, 'eta': 3 } # To optimize, we pass the function to the SMAC-object smac = SMAC4MF(scenario=scenario, rng=np.random.RandomState(42), tae_runner=mlp_from_cfg, intensifier_kwargs=intensifier_kwargs) # Example call of the function with default values # It returns: Status, Cost, Runtime, Additional Infos def_value = smac.get_tae_runner().run( config=cs.get_default_configuration(), budget=max_epochs, seed=0)[1] print('Value for default configuration: %.4f' % def_value) # Start optimization try: incumbent = smac.optimize() finally: incumbent = smac.solver.incumbent inc_value = smac.get_tae_runner().run(config=incumbent, budget=max_epochs, seed=0)[1] print('Optimized Value: %.4f' % inc_value)
# Build Configuration Space which defines all parameters and their ranges cs = ConfigurationSpace() x0 = UniformFloatHyperparameter("x0", -5, 10, default_value=-3) x1 = UniformFloatHyperparameter("x1", -5, 10, default_value=-4) cs.add_hyperparameters([x0, x1]) # Scenario object scenario = Scenario({ "run_obj": "quality", # we optimize quality (alternatively runtime) "runcount-limit": 10, # max. number of function evaluations; for this example set to a low number "cs": cs, # configuration space "deterministic": "true" }) # Example call of the function # It returns: Status, Cost, Runtime, Additional Infos def_value = rosenbrock_2d(cs.get_default_configuration()) print("Default Value: %.2f" % def_value) # Optimize, using a SMAC-object print("Optimizing! Depending on your machine, this might take a few minutes.") smac = SMAC4BO( scenario=scenario, rng=np.random.RandomState(42), tae_runner=rosenbrock_2d, ) smac.optimize()
# some hyperparameters depend on others #This work is done by 'if condition' in the called function # Scenario object scenario = Scenario({ "run_obj": "quality", # we optimize quality (alternatively runtime) "runcount-limit": 500, # max. number of function evaluations; for this example set to a low number "cs": cs, # configuration space "deterministic": "true" }) # Example call of the function # It returns: Status, Cost, Runtime, Additional Infos def_value = LR_from_cfg(cs.get_default_configuration()) print("Default Value: %.2f" % (def_value)) # Optimize, using a SMAC-object print("Optimizing! Depending on your machine, this might take a few minutes.") smac = SMAC4HPO(scenario=scenario, rng=np.random.RandomState(42), tae_runner=LR_from_cfg) a_time = time.process_time() incumbent = smac.optimize() b_time = time.process_time() print("+++++++++++++++++++++++") print("Optimization finished. CPU time consumed: %s" % (a_time - b_time)) print("+++++++++++++++++++++++")
100, 100000, default=1000) cs.add_hyperparameter(max_num_nodes) # SMAC scenario oject scenario = Scenario({ "run_obj": "quality", # we optimize quality (alternative runtime) "runcount-limit": 400, # at most 200 function evaluations "cs": cs, # configuration space "deterministic": "true", "memory_limit": 1024, }) # Optimize smac = SMAC(scenario=scenario, rng=np.random.RandomState(42), tae_runner=rfr) # example call of the function # it returns: Status, Cost, Runtime, Additional Infos def_value = smac.solver.intensifier.tae_runner.run( cs.get_default_configuration(), 1)[1] print("Default Value: %.2f" % (def_value)) try: incumbent = smac.optimize() finally: incumbent = smac.solver.incumbent inc_value = smac.solver.intensifier.tae_runner.run(incumbent, 1)[1] print("Optimized Value: %.2f" % (inc_value))