def optimize(self):
cs = ConfigurationSpace()
if 'optimizer' in self.params:
optimizer = CategoricalHyperparameter('optimizer',
self.params['optimizer'])
cs.add_hyperparameter(optimizer)
scenario = Scenario({
"run_obj": "quality",
"runcount-limit": 5,
"cutoff-time": 10,
"cs": cs,
"deterministic": "true"
})
print(cs.get_default_configuration())
def_value = self.svm_from_config(cs.get_default_configuration())
smac = SMAC4HPO(scenario=scenario,
rng=np.random.RandomState(42),
tae_runner=self.svm_from_config)
incumbent = smac.optimize()
__params = incumbent.get_dictionary()
inc_value = self.svm_from_config(incumbent)
print(__params)
model = create_model(**__params)
hist = model.fit(self.x_train, self.y_train, batch_size=128, epochs=6)
time = smac.stats.wallclock_time_used
loss, accuracy, f1_score, precision, recall = model.evaluate(
self.x_test, self.y_test, verbose=0)
del smac
self.result = {
'accuracy': accuracy,
'time': time,
'best_params': __params
}
def optimize():
# We load the iris-dataset (a widely used benchmark)
iris = datasets.load_iris()
#logger = logging.getLogger("SVMExample")
logging.basicConfig(level=logging.INFO) # logging.DEBUG for debug output
# Build Configuration Space which defines all parameters and their ranges
cs = ConfigurationSpace()
# We define a few possible types of SVM-kernels and add them as "kernel" to our cs
kernel = CategoricalHyperparameter("kernel", ["linear", "rbf", "poly", "sigmoid"], default="poly")
cs.add_hyperparameter(kernel)
# There are some hyperparameters shared by all kernels
C = UniformFloatHyperparameter("C", 0.001, 1000.0, default=1.0)
shrinking = CategoricalHyperparameter("shrinking", ["true", "false"], default="true")
cs.add_hyperparameters([C, shrinking])
# Others are kernel-specific, so we can add conditions to limit the searchspace
degree = UniformIntegerHyperparameter("degree", 1, 5, default=3) # Only used by kernel poly
coef0 = UniformFloatHyperparameter("coef0", 0.0, 10.0, default=0.0) # poly, sigmoid
cs.add_hyperparameters([degree, coef0])
use_degree = InCondition(child=degree, parent=kernel, values=["poly"])
use_coef0 = InCondition(child=coef0, parent=kernel, values=["poly", "sigmoid"])
cs.add_conditions([use_degree, use_coef0])
# This also works for parameters that are a mix of categorical and values from a range of numbers
# For example, gamma can be either "auto" or a fixed float
gamma = CategoricalHyperparameter("gamma", ["auto", "value"], default="auto") # only rbf, poly, sigmoid
gamma_value = UniformFloatHyperparameter("gamma_value", 0.0001, 8, default=1)
cs.add_hyperparameters([gamma, gamma_value])
# We only activate gamma_value if gamma is set to "value"
cs.add_condition(InCondition(child=gamma_value, parent=gamma, values=["value"]))
# And again we can restrict the use of gamma in general to the choice of the kernel
cs.add_condition(InCondition(child=gamma, parent=kernel, values=["rbf", "poly", "sigmoid"]))
# Scenario object
scenario = Scenario("test/test_files/svm_scenario.txt")
# Example call of the function
# It returns: Status, Cost, Runtime, Additional Infos
def_value = svm_from_cfg(cs.get_default_configuration())
print("Default Value: %.2f" % (def_value))
# Optimize, using a SMAC-object
print("Optimizing! Depending on your machine, this might take a few minutes.")
smac = SMAC(scenario=scenario, rng=np.random.RandomState(42),
tae_runner=svm_from_cfg)
incumbent = smac.optimize()
inc_value = svm_from_cfg(incumbent)
print("Optimized Value: %.2f" % (inc_value))
def main_loop(problem):
logging.basicConfig(level=logging.INFO) # logging.DEBUG for debug output
cs = ConfigurationSpace()
n_neighbors = UniformIntegerHyperparameter("n_neighbors", 2,10, default_value=5)
cs.add_hyperparameter(n_neighbors)
weights = CategoricalHyperparameter("weights", ["uniform","distance"], default_value="uniform")
algorithm = CategoricalHyperparameter("algorithm", ["ball_tree", "kd_tree","brute","auto"], default_value="auto")
cs.add_hyperparameters([weights, algorithm])
leaf_size = UniformIntegerHyperparameter("leaf_size", 1, 100, default_value=50)
cs.add_hyperparameter(leaf_size)
use_leaf_size= InCondition(child=leaf_size, parent=algorithm, values=["ball_tree","kd_tree"])
cs.add_condition(use_leaf_size)
p = UniformIntegerHyperparameter("p", 1,3, default_value=2)
cs.add_hyperparameter(p)
# Scenario object
max_eval=100000
scenario = Scenario({"run_obj": "quality", # we optimize quality (alternatively runtime)
"runcount-limit": max_eval, # maximum function evaluations
"cs": cs, # configuration space
"shared_model": True,
"output_dir": "/home/naamah/Documents/CatES/result_All/smac/KNN/run_{}_{}_{}".format(max_eval,
datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d_%H:%M:%S'),
problem)
# "output_dir": "/home/naamah/Documents/CatES/result_All/smac/KNN/{}/run_{}_{}".format(problem,max_eval, datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d_%H:%M:%S_%f')),
# "input_psmac_dirs":"/home/naamah/Documents/CatES/result_All/",
# "deterministic": "true"
})
# Example call of the function
# It returns: Status, Cost, Runtime, Additional Infos
def_value = svm_from_cfg(cs.get_default_configuration())
print("Default Value: %.2f" % (def_value))
# Optimize, using a SMAC-object
print("Optimizing! Depending on your machine, this might take a few minutes.")
smac = SMAC(scenario=scenario,tae_runner=svm_from_cfg)
incumbent = smac.optimize()
inc_value = svm_from_cfg(incumbent)
print("Optimized Value: %.2f" % (inc_value))
return (incumbent)
# main_loop()
def test_illegal_input(self):
"""
Testing illegal input in smbo
"""
cs = ConfigurationSpace()
cs.add_hyperparameter(UniformFloatHyperparameter('test', 1, 10, 5))
scen = Scenario({'run_obj': 'quality', 'cs': cs})
stats = Stats(scen)
# Recorded runs but no incumbent.
stats.ta_runs = 10
smac = SMAC(scen, stats=stats, rng=np.random.RandomState(42))
self.assertRaises(ValueError, smac.optimize)
# Incumbent but no recoreded runs.
incumbent = cs.get_default_configuration()
smac = SMAC(scen, restore_incumbent=incumbent,
rng=np.random.RandomState(42))
self.assertRaises(ValueError, smac.optimize)
scenario = Scenario({
"run_obj": "quality", # we optimize quality (alternative runtime)
"runcount-limit": 200, # at most 200 function evaluations
"cs": cs, # configuration space
"deterministic": "true",
#"tuner-timeout" : 1,
#"wallclock_limit": 2
})
stats = Stats(scenario)
# register function to be optimize
taf = ExecuteTAFunc(rosenbrock_4d, stats)
# example call of the function
# it returns: Status, Cost, Runtime, Additional Infos
def_value = taf.run(cs.get_default_configuration())[1]
print("Default Value: %.2f" % (def_value))
# Optimize
smbo = SMBO(scenario=scenario,
tae_runner=taf,
stats=stats,
rng=np.random.RandomState(42))
try:
smbo.run(max_iters=999)
finally:
smbo.stats.print_stats()
print("Final Incumbent: %s" % (smbo.incumbent))
inc_value = taf.run(smbo.incumbent)[1]
print("Optimized Value: %.2f" % (inc_value))
def main_loop(problem):
logging.basicConfig(level=logging.INFO) # logging.DEBUG for debug output
cs = ConfigurationSpace()
n_estimators = UniformIntegerHyperparameter("n_estimators",
5,
50,
default_value=10)
#criterion = CategoricalHyperparameter("criterion", ["mse", "mae"], default_value="mse")
min_samples_split = UniformIntegerHyperparameter("min_samples_split",
2,
20,
default_value=2)
min_samples_leaf = UniformIntegerHyperparameter("min_samples_leaf",
1,
20,
default_value=1)
min_weight_fraction_leaf = UniformFloatHyperparameter(
"min_weight_fraction_leaf", 0.0, 0.5, default_value=0.0)
max_leaf_nodes = UniformIntegerHyperparameter("max_leaf_nodes",
10,
1000,
default_value=100)
min_impurity_decrease = UniformFloatHyperparameter("min_impurity_decrease",
0.0,
0.5,
default_value=0.0)
warm_start = CategoricalHyperparameter("warm_start", ["true", "false"],
default_value="false")
cs.add_hyperparameters([
n_estimators, min_weight_fraction_leaf, min_samples_split,
min_samples_leaf, max_leaf_nodes, warm_start, min_impurity_decrease
])
max_features = CategoricalHyperparameter(
"max_features", ["auto", "log2", "sqrt", "int", "None", "float"],
default_value="auto") # only rbf, poly, sigmoid
max_features_int = UniformIntegerHyperparameter("max_features_int",
2,
len(X[0]),
default_value=5)
max_features_float = UniformFloatHyperparameter("max_features_float",
0.0,
0.9,
default_value=0.0)
cs.add_hyperparameters(
[max_features, max_features_int, max_features_float])
use_max_features_int = InCondition(child=max_features_int,
parent=max_features,
values=["int"])
use_max_features_float = InCondition(child=max_features_float,
parent=max_features,
values=["float"])
cs.add_conditions([use_max_features_int, use_max_features_float])
max_depth = CategoricalHyperparameter("max_depth", ["None", "value"],
default_value="None")
max_depth_value = UniformIntegerHyperparameter("max_depth_value",
2,
20,
default_value=5)
cs.add_hyperparameters([max_depth, max_depth_value])
cs.add_condition(
InCondition(child=max_depth_value, parent=max_depth, values=["value"]))
random_state = CategoricalHyperparameter("random_state", ["None", "value"],
default_value="None")
random_state_value = UniformIntegerHyperparameter("random_state_value",
1,
20,
default_value=1)
cs.add_hyperparameters([random_state, random_state_value])
cs.add_condition(
InCondition(child=random_state_value,
parent=random_state,
values=["value"]))
with open("/home/naamah/Documents/CatES/result_All/X1.p", "rb") as fp:
X = pickle.load(fp)
# Scenario object
max_eval = 100000
scenario = Scenario({
"run_obj":
"quality", # we optimize quality (alternatively runtime)
"runcount-limit":
max_eval, # maximum function evaluations
"cs":
cs, # configuration space
"shared_model":
True,
"output_dir":
"/home/naamah/Documents/CatES/result_All/smac/RF/run_{}_{}_{}".format(
max_eval,
datetime.datetime.fromtimestamp(
time.time()).strftime('%Y-%m-%d_%H:%M:%S'), problem),
"input_psmac_dirs":
"/home/naamah/Documents/CatES/result_All/smac/psmac",
"deterministic":
"False"
})
def_value = svm_from_cfg(cs.get_default_configuration())
print("Default Value: %.2f" % (def_value))
# Optimize, using a SMAC-object
print(
"Optimizing! Depending on your machine, this might take a few minutes."
)
smac = SMAC(scenario=scenario, tae_runner=svm_from_cfg)
incumbent = smac.optimize()
inc_value = svm_from_cfg(incumbent)
print("Optimized Value: %.2f" % (inc_value))
return (incumbent)
# main_loop()
# Scenario object
scenario = Scenario({
"run_obj": "quality", # we optimize quality
# (alternatively runtime)
"runcount-limit": 10,
"cs": cs, # configuration space
"deterministic": "false",
"shared_model": True,
"input_psmac_dirs": "smac-output-learch",
"cutoff_time": 9000,
"wallclock_limit": 'inf'
})
# Example call of the function
# It returns: Status, Cost, Runtime, Additional Infos
def_value = learch(cs.get_default_configuration())
print("Default Value: %.2f" % def_value)
# Optimize, using a SMAC-object
print("Optimizing! Depending on your machine, this might take a few minutes.")
smac = SMAC4HPO(scenario=scenario,
rng=np.random.RandomState(42),
tae_runner=learch)
# Start optimization
try:
incumbent = smac.optimize()
finally:
incumbent = smac.solver.incumbent
default_value=4)
mutation_prob = UniformFloatHyperparameter("mutation_prob",
0.1,
0.9,
default_value=0.5)
cs.add_hyperparameters(
[num_generations, number_chromosomes, num_parents_mating, mutation_prob])
# Scenario object
scenario = Scenario({
"run_obj": "quality", # we optimize quality (alternatively runtime)
"runcount-limit":
10, # max. number of function evaluations; for this example set to a low number
"cs": cs, # configuration space
"deterministic": "true"
})
# Example call of the function
# It returns: Status, Cost, Runtime, Additional Infos
def_value = gawd(cs.get_default_configuration())
# Optimize, using a SMAC-object
print("Optimizing! Depending on your machine, this might take a few minutes.")
smac = SMAC4BO(
scenario=scenario,
rng=np.random.RandomState(42),
tae_runner=gawd,
)
smac.optimize()
"eta": 3
}
# To optimize, we pass the function to the SMAC-object
smac = SMAC4MF(
scenario=scenario,
rng=np.random.RandomState(42),
tae_runner=mlp_from_cfg,
intensifier_kwargs=intensifier_kwargs,
)
tae = smac.get_tae_runner()
# Example call of the function with default values
# It returns: Status, Cost, Runtime, Additional Infos
def_value = tae.run(config=cs.get_default_configuration(),
budget=max_epochs,
seed=0)[1]
print("Value for default configuration: %.4f" % def_value)
# Start optimization
try:
incumbent = smac.optimize()
finally:
incumbent = smac.solver.incumbent
inc_value = tae.run(config=incumbent, budget=max_epochs, seed=0)[1]
print("Optimized Value: %.4f" % inc_value)
scenario = Scenario({"run_obj": "quality", # we optimize quality (alternative runtime)
"runcount-limit": 50, # maximum number of function evaluations
"cs": cs, # configuration space
"deterministic": "true",
"memory_limit": 3072, # adapt this to reasonable value for your hardware
"shared_model": True,
"input_psmac_dirs": "smac3-output*"
})
# To optimize, we pass the function to the SMAC-object
smac = SMAC(scenario=scenario, rng=np.random.RandomState(42),
tae_runner=rf_from_cfg)
# Example call of the function with default values
# It returns: Status, Cost, Runtime, Additional Infos
def_value = smac.get_tae_runner().run(cs.get_default_configuration(), 1)[1]
print("Value for default configuration: %.2f" % (def_value))
# Start optimization
try:
incumbent = smac.optimize()
finally:
incumbent = smac.solver.incumbent
incumbent._values
mlflow.log_param("city",city)
mlflow.log_param("model","rf")
all_feats = ""
for elem in features:
if all_feats == "":
# Scenario object
scenario = Scenario({
"run_obj": "quality", # we optimize quality
# (alternatively runtime)
"runcount-limit": 10,
"cs": cs, # configuration space
"deterministic": "false",
"shared_model": True,
"input_psmac_dirs": "smac-output-maxEnt",
"cutoff_time": 9000,
"wallclock_limit": 'inf'
})
# Example call of the function
# It returns: Status, Cost, Runtime, Additional Infos
def_value = maxEnt(cs.get_default_configuration())
print("Default Value: %.2f" % def_value)
# Optimize, using a SMAC-object
print("Optimizing! Depending on your machine, this might take a few minutes.")
smac = SMAC4HPO(scenario=scenario,
rng=np.random.RandomState(42),
tae_runner=maxEnt)
# Start optimization
try:
incumbent = smac.optimize()
finally:
incumbent = smac.solver.incumbent
for i in range(total_features):
param_id = UniformIntegerHyperparameter(
str(i), 0, 1, default_value=0) # Only used by kernel poly
cs.add_hyperparameter(param_id)
# Scenario object
scenario = Scenario({
"run_obj": "quality", # we optimize quality (alternatively runtime)
"runcount-limit": 200, # maximum function evaluations
"cs": cs, # configuration space
"deterministic": "true"
})
# Example call of the function
# It returns: Status, Cost, Runtime, Additional Infos
def_value = simulator_fun(cs.get_default_configuration())
print("Default Value: %.2f" % (def_value))
# Optimize, using a SMAC-object
print("Optimizing! Depending on your machine, this might take a few minutes.")
total_value = np.zeros(200)
for i in range(5):
smac = SMAC(scenario=scenario,
rng=np.random.RandomState(42),
tae_runner=simulator_fun)
incumbent = smac.optimize()
print(smac.trajectory)
idx = 0
for j in range(200):
if (idx < len(smac.trajectory)):
1.0,
default_value=0.1,
log=True)
max_features = UniformIntegerHyperparameter("max_features",
2,
10,
default_value=4)
cs.add_hyperparameters([min_samples_split, max_features])
subsample = UniformFloatHyperparameter("subsample",
0.5,
1,
default_value=0.8)
cs.add_hyperparameter(subsample)
cfg = cs.get_default_configuration()
clf = GradientBoostingClassifier(**cfg,
random_state=0).fit(X_train, y_train)
def_test_score = 1 - clf.score(X_test, y_test)
print("Default cross validation score: %.2f" % (xgboost_from_cfg(cfg)))
print("Default test score: %.2f" % def_test_score)
# scenario object
scenario = Scenario({
"run_obj": "quality",
"runcount-limit": 100,
"cs": cs,
# the evaluations are not deterministic, we need to repeat each
# configuration several times and take the mean value of these repetitions
"deterministic": "false",
cs = ConfigurationSpace()
max_depth = UniformIntegerHyperparameter("max_depth", 1, 10, default_value=3)
cs.add_hyperparameter(max_depth)
learning_rate = UniformFloatHyperparameter("learning_rate", 0.01, 1.0, default_value=1.0, log=True)
cs.add_hyperparameter(learning_rate)
min_samples_split = UniformFloatHyperparameter("min_samples_split", 0.01, 1.0, default_value=0.1, log=True)
max_features = UniformIntegerHyperparameter("max_features", 2, 10, default_value=4)
cs.add_hyperparameters([min_samples_split, max_features])
subsample = UniformFloatHyperparameter("subsample", 0.5, 1, default_value=0.8)
cs.add_hyperparameter(subsample)
print("Default cross validation score: %.2f" % (xgboost_from_cfg(cs.get_default_configuration())))
cfg = cs.get_default_configuration()
clf = GradientBoostingClassifier(**cfg, random_state=0).fit(X_train, y_train)
def_test_score = 1 - clf.score(X_test, y_test)
print("Default test score: %.2f" % def_test_score)
# scenario object
scenario = Scenario({
"run_obj": "quality",
"runcount-limit": 100,
"cs": cs,
# the evaluations are not deterministic, we need to repeat each
# configuration several times and take the mean value of these repetitions
"deterministic": "false",
"wallclock_limit": 120,
"maxR": 3, # Each configuration will be evaluated maximal 3 times with various seeds
cs.add_hyperparameters([learning_rate, step_size_scalar, l2_regularizer, N])
# Scenario object
scenario = Scenario({"run_obj": "quality", # we optimize quality
# (alternatively runtime)
"runcount-limit": 2,
"cs": cs, # configuration space
"deterministic": "false",
"shared_model": True,
"input_psmac_dirs": "smac-output-learch",
"cutoff_time": 9000,
"wallclock_limit": 'inf'
})
# Example call of the function
# It returns: Status, Cost, Runtime, Additional Infos
def_value = learch_variant(cs.get_default_configuration())
print("Default Value: %.2f" % def_value)
# Optimize, using a SMAC-object
print("Optimizing! Depending on your machine, this might take a few minutes.")
smac = SMAC4HPO(scenario=scenario,
rng=np.random.RandomState(42),
tae_runner=learch_variant)
# Start optimization
try:
incumbent = smac.optimize()
finally:
incumbent = smac.solver.incumbent
"cs": cs, # configuration space
"deterministic": False,
"shared_model": True,
"output_dir": "smac3-output/",
"input_psmac_dirs": "smac3-output/run_*",
"instances": INSTANCES
})
i = int(sys.argv[1])
# Optimize, using a SMAC-object
print("Optimizing! Depending on your machine, this might take a few minutes.")
smac = BOHB4HPO(scenario=scenario,
rng=np.random.RandomState(i),
tae_runner=gadma_from_cfg,
run_id=i,
intensifier_kwargs=intensifier_kwargs)
incumbent = smac.optimize()
def_costs = []
for i in INSTANCES:
cost = smac.get_tae_runner().run(cs.get_default_configuration(), i[0])[1]
def_costs.append(cost)
print("Value for default configuration: %.4f" % (np.mean(def_costs)))
inc_costs = []
for i in INSTANCES:
cost = smac.get_tae_runner().run(incumbent, i[0])[1]
inc_costs.append(cost)
print("Optimized Value: %.4f" % (np.mean(inc_costs)))
def main_loop(problem):
#logger = logging.getLogger("SVMExample")
logging.basicConfig(level=logging.INFO) # logging.DEBUG for debug output
cs = ConfigurationSpace()
kernel = CategoricalHyperparameter("kernel", ["rbf", "sigmoid"],
default_value="rbf")
cs.add_hyperparameter(kernel)
C = UniformFloatHyperparameter("C", 1, 1000.0, default_value=900.0)
shrinking = CategoricalHyperparameter("shrinking", ["true", "false"],
default_value="false")
cs.add_hyperparameters([C, shrinking])
#degree = UniformIntegerHyperparameter("degree", 1, 3 ,default_value=3) # Only used by kernel poly
coef0 = UniformFloatHyperparameter("coef0", 0.0, 1.0,
default_value=0.0) # poly, sigmoid
#cs.add_hyperparameters([degree, coef0])
cs.add_hyperparameter(coef0)
#use_degree = InCondition(child=degree, parent=kernel, values=["poly"])
use_coef0 = InCondition(child=coef0, parent=kernel, values=["sigmoid"])
#cs.add_conditions([use_degree, use_coef0])
cs.add_condition(use_coef0)
gamma = CategoricalHyperparameter(
"gamma", ["auto", "value"],
default_value="auto") # only rbf, poly, sigmoid
gamma_value = UniformFloatHyperparameter("gamma_value",
0.001,
8,
default_value=1)
cs.add_hyperparameters([gamma, gamma_value])
cs.add_condition(
InCondition(child=gamma_value, parent=gamma, values=["value"]))
cs.add_condition(
InCondition(child=gamma, parent=kernel, values=["rbf", "sigmoid"]))
epsilon = UniformFloatHyperparameter("epsilon",
0.001,
5.0,
default_value=0.1)
cs.add_hyperparameter(epsilon)
# Scenario object
max_eval = 100000
scenario = Scenario({
"run_obj":
"quality", # we optimize quality (alternatively runtime)
"runcount-limit":
max_eval, # maximum function evaluations
"cs":
cs, # configuration space
"shared_model":
True,
"output_dir":
"/home/naamah/Documents/CatES/result_All/smac/svm/run_{}_{}_{}".format(
max_eval,
datetime.datetime.fromtimestamp(
time.time()).strftime('%Y-%m-%d_%H:%M:%S'), problem),
# "input_psmac_dirs": "/home/naamah/Documents/CatES/result_All/smac/svm/{}/run_{}_{}/*".format(problem,max_eval, datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d_%H:%M:%S_%f')),
"deterministic":
"False"
#"instance_file":"/home/naamah/Documents/CatES/result_All",
#"test_instance_file":"/home/naamah/Documents/CatES/result_All"
})
# Example call of the function
# It returns: Status, Cost, Runtime, Additional Infos
def_value = svm_from_cfg(cs.get_default_configuration())
print("Default Value: %.2f" % (def_value))
# Optimize, using a SMAC-object
print(
"Optimizing! Depending on your machine, this might take a few minutes."
)
smac = SMAC(scenario=scenario, tae_runner=svm_from_cfg)
incumbent = smac.optimize()
inc_value = svm_from_cfg(incumbent)
print("Optimized Value: %.2f" % (inc_value))
return (incumbent)
def quant_post_hpo(
executor,
place,
model_dir,
quantize_model_path,
train_sample_generator=None,
eval_sample_generator=None,
train_dataloader=None,
eval_dataloader=None,
eval_function=None,
model_filename=None,
params_filename=None,
save_model_filename='__model__',
save_params_filename='__params__',
scope=None,
quantizable_op_type=["conv2d", "depthwise_conv2d", "mul"],
is_full_quantize=False,
weight_bits=8,
activation_bits=8,
weight_quantize_type=['channel_wise_abs_max'],
algo=["KL", "hist", "avg", "mse"],
bias_correct=[True, False],
hist_percent=[0.98, 0.999], ### uniform sample in list.
batch_size=[10, 30], ### uniform sample in list.
batch_num=[10, 30], ### uniform sample in list.
optimize_model=False,
is_use_cache_file=False,
cache_dir="./temp_post_training",
runcount_limit=30):
"""
The function utilizes static post training quantization method to
quantize the fp32 model. It uses calibrate data to calculate the
scale factor of quantized variables, and inserts fake quantization
and dequantization operators to obtain the quantized model.
Args:
executor(paddle.static.Executor): The executor to load, run and save the
quantized model.
place(paddle.CPUPlace or paddle.CUDAPlace): This parameter represents
the executor run on which device.
model_dir(str): The path of fp32 model that will be quantized, and
the model and params that saved by ``paddle.static.io.save_inference_model``
are under the path.
quantize_model_path(str): The path to save quantized model using api
``paddle.static.io.save_inference_model``.
train_sample_generator(Python Generator): The sample generator provides
calibrate data for DataLoader, and it only returns a sample every time.
eval_sample_generator(Python Generator): The sample generator provides
evalution data for DataLoader, and it only returns a sample every time.
model_filename(str, optional): The name of model file. If parameters
are saved in separate files, set it as 'None'. Default: 'None'.
params_filename(str, optional): The name of params file.
When all parameters are saved in a single file, set it
as filename. If parameters are saved in separate files,
set it as 'None'. Default : 'None'.
save_model_filename(str): The name of model file to save the quantized inference program. Default: '__model__'.
save_params_filename(str): The name of file to save all related parameters.
If it is set None, parameters will be saved in separate files. Default: '__params__'.
scope(paddle.static.Scope, optional): The scope to run program, use it to load
and save variables. If scope is None, will use paddle.static.global_scope().
quantizable_op_type(list[str], optional): The list of op types
that will be quantized. Default: ["conv2d", "depthwise_conv2d",
"mul"].
is_full_quantize(bool): if True, apply quantization to all supported quantizable op type.
If False, only apply quantization to the input quantizable_op_type. Default is False.
weight_bits(int, optional): quantization bit number for weights.
activation_bits(int): quantization bit number for activation.
weight_quantize_type(str): quantization type for weights,
support 'abs_max' and 'channel_wise_abs_max'. Compared to 'abs_max',
the model accuracy is usually higher when using 'channel_wise_abs_max'.
optimize_model(bool, optional): If set optimize_model as True, it applies some
passes to optimize the model before quantization. So far, the place of
executor must be cpu it supports fusing batch_norm into convs.
is_use_cache_file(bool): This param is deprecated.
cache_dir(str): This param is deprecated.
runcount_limit(int): max. number of model quantization.
Returns:
None
"""
global g_quant_config
g_quant_config = QuantConfig(
executor, place, model_dir, quantize_model_path, algo, hist_percent,
bias_correct, batch_size, batch_num, train_sample_generator,
eval_sample_generator, train_dataloader, eval_dataloader,
eval_function, model_filename, params_filename, save_model_filename,
save_params_filename, scope, quantizable_op_type, is_full_quantize,
weight_bits, activation_bits, weight_quantize_type, optimize_model,
is_use_cache_file, cache_dir)
cs = ConfigurationSpace()
hyper_params = []
if 'hist' in algo:
hist_percent = UniformFloatHyperparameter(
"hist_percent",
hist_percent[0],
hist_percent[1],
default_value=hist_percent[0])
hyper_params.append(hist_percent)
if len(algo) > 1:
algo = CategoricalHyperparameter("algo", algo, default_value=algo[0])
hyper_params.append(algo)
else:
algo = algo[0]
if len(bias_correct) > 1:
bias_correct = CategoricalHyperparameter("bias_correct",
bias_correct,
default_value=bias_correct[0])
hyper_params.append(bias_correct)
else:
bias_correct = bias_correct[0]
if len(weight_quantize_type) > 1:
weight_quantize_type = CategoricalHyperparameter("weight_quantize_type", \
weight_quantize_type, default_value=weight_quantize_type[0])
hyper_params.append(weight_quantize_type)
else:
weight_quantize_type = weight_quantize_type[0]
if len(batch_size) > 1:
batch_size = UniformIntegerHyperparameter("batch_size",
batch_size[0],
batch_size[1],
default_value=batch_size[0])
hyper_params.append(batch_size)
else:
batch_size = batch_size[0]
if len(batch_num) > 1:
batch_num = UniformIntegerHyperparameter("batch_num",
batch_num[0],
batch_num[1],
default_value=batch_num[0])
hyper_params.append(batch_num)
else:
batch_num = batch_num[0]
if len(hyper_params) == 0:
quant_post( \
executor=g_quant_config.executor, \
scope=g_quant_config.scope, \
model_dir=g_quant_config.float_infer_model_path, \
quantize_model_path=g_quant_model_cache_path, \
sample_generator=g_quant_config.train_sample_generator, \
data_loader=g_quant_config.train_dataloader,
model_filename=g_quant_config.model_filename, \
params_filename=g_quant_config.params_filename, \
save_model_filename=g_quant_config.save_model_filename, \
save_params_filename=g_quant_config.save_params_filename, \
quantizable_op_type=g_quant_config.quantizable_op_type, \
activation_quantize_type="moving_average_abs_max", \
weight_quantize_type=weight_quantize_type, \
algo=algo, \
hist_percent=hist_percent, \
bias_correction=bias_correct, \
batch_size=batch_size, \
batch_nums=batch_num)
return
cs.add_hyperparameters(hyper_params)
scenario = Scenario({
"run_obj": "quality", # we optimize quality (alternative runtime)
"runcount-limit":
runcount_limit, # max. number of function evaluations; for this example set to a low number
"cs": cs, # configuration space
"deterministic": "True",
"limit_resources": "False",
"memory_limit":
4096 # adapt this to reasonable value for your hardware
})
# To optimize, we pass the function to the SMAC-object
smac = SMAC4HPO(scenario=scenario,
rng=np.random.RandomState(42),
tae_runner=quantize)
# Example call of the function with default values
# It returns: Status, Cost, Runtime, Additional Infos
def_value = smac.get_tae_runner().run(cs.get_default_configuration(), 1)[1]
print("Value for default configuration: %.8f" % def_value)
# Start optimization
try:
incumbent = smac.optimize()
finally:
incumbent = smac.solver.incumbent
inc_value = smac.get_tae_runner().run(incumbent, 1)[1]
print("Optimized Value: %.8f" % inc_value)
print("quantize completed")
'instance_order': 'shuffle_once'
}
# To optimize, we pass the function to the SMAC-object
smac = ROAR(scenario=scenario,
rng=np.random.RandomState(42),
tae_runner=mlp_from_cfg,
intensifier=SuccessiveHalving,
intensifier_kwargs=intensifier_kwargs,
initial_design=RandomConfigurations,
n_jobs=4)
# Example call of the function with default values
# It returns: Status, Cost, Runtime, Additional Infos
def_value = smac.get_tae_runner().run(
config=cs.get_default_configuration(), instance='1', budget=25,
seed=0)[1]
print("Value for default configuration: %.4f" % def_value)
# Start optimization
try:
incumbent = smac.optimize()
finally:
incumbent = smac.solver.incumbent
inc_value = smac.get_tae_runner().run(config=incumbent,
instance='1',
budget=25,
seed=0)[1]
print("Optimized Value: %.4f" % inc_value)
intensifier_kwargs = {
'initial_budget': 5,
'max_budget': max_epochs,
'eta': 3
}
# To optimize, we pass the function to the SMAC-object
smac = SMAC4MF(scenario=scenario,
rng=np.random.RandomState(42),
tae_runner=mlp_from_cfg,
intensifier_kwargs=intensifier_kwargs)
# Example call of the function with default values
# It returns: Status, Cost, Runtime, Additional Infos
def_value = smac.get_tae_runner().run(
config=cs.get_default_configuration(), budget=max_epochs, seed=0)[1]
print('Value for default configuration: %.4f' % def_value)
# Start optimization
try:
incumbent = smac.optimize()
finally:
incumbent = smac.solver.incumbent
inc_value = smac.get_tae_runner().run(config=incumbent,
budget=max_epochs,
seed=0)[1]
print('Optimized Value: %.4f' % inc_value)
# Build Configuration Space which defines all parameters and their ranges
cs = ConfigurationSpace()
x0 = UniformFloatHyperparameter("x0", -5, 10, default_value=-3)
x1 = UniformFloatHyperparameter("x1", -5, 10, default_value=-4)
cs.add_hyperparameters([x0, x1])
# Scenario object
scenario = Scenario({
"run_obj": "quality", # we optimize quality (alternatively runtime)
"runcount-limit":
10, # max. number of function evaluations; for this example set to a low number
"cs": cs, # configuration space
"deterministic": "true"
})
# Example call of the function
# It returns: Status, Cost, Runtime, Additional Infos
def_value = rosenbrock_2d(cs.get_default_configuration())
print("Default Value: %.2f" % def_value)
# Optimize, using a SMAC-object
print("Optimizing! Depending on your machine, this might take a few minutes.")
smac = SMAC4BO(
scenario=scenario,
rng=np.random.RandomState(42),
tae_runner=rosenbrock_2d,
)
smac.optimize()
# some hyperparameters depend on others
#This work is done by 'if condition' in the called function
# Scenario object
scenario = Scenario({
"run_obj": "quality", # we optimize quality (alternatively runtime)
"runcount-limit":
500, # max. number of function evaluations; for this example set to a low number
"cs": cs, # configuration space
"deterministic": "true"
})
# Example call of the function
# It returns: Status, Cost, Runtime, Additional Infos
def_value = LR_from_cfg(cs.get_default_configuration())
print("Default Value: %.2f" % (def_value))
# Optimize, using a SMAC-object
print("Optimizing! Depending on your machine, this might take a few minutes.")
smac = SMAC4HPO(scenario=scenario,
rng=np.random.RandomState(42),
tae_runner=LR_from_cfg)
a_time = time.process_time()
incumbent = smac.optimize()
b_time = time.process_time()
print("+++++++++++++++++++++++")
print("Optimization finished. CPU time consumed: %s" % (a_time - b_time))
print("+++++++++++++++++++++++")
100,
100000,
default=1000)
cs.add_hyperparameter(max_num_nodes)
# SMAC scenario oject
scenario = Scenario({
"run_obj": "quality", # we optimize quality (alternative runtime)
"runcount-limit": 400, # at most 200 function evaluations
"cs": cs, # configuration space
"deterministic": "true",
"memory_limit": 1024,
})
# Optimize
smac = SMAC(scenario=scenario, rng=np.random.RandomState(42), tae_runner=rfr)
# example call of the function
# it returns: Status, Cost, Runtime, Additional Infos
def_value = smac.solver.intensifier.tae_runner.run(
cs.get_default_configuration(), 1)[1]
print("Default Value: %.2f" % (def_value))
try:
incumbent = smac.optimize()
finally:
incumbent = smac.solver.incumbent
inc_value = smac.solver.intensifier.tae_runner.run(incumbent, 1)[1]
print("Optimized Value: %.2f" % (inc_value))
