def fit_xcorr_bo(detector_image, detector_pitch, as_to_m, src_peak_photons):
# Assume npixels = 512
model = Model()
# Use cross correlation to estimate major source location
model.set_model_params([0,0,1.,0,0,0])
psf_template=model.model_image()[256-50:256+50,256-50:256+50]
psf_template -= psf_template.mean()
corr = signal.correlate2d(detector_image, psf_template, boundary='symm', mode='same')
max_loc = peak_local_max(corr,num_peaks=1)[0]
# Estimate major peak
est_flux = detector_image.max(axis=None)*1. / src_peak_photons
model_bsf = [(max_loc[0] - 256)*detector_pitch/as_to_m, (max_loc[1] - 256)*detector_pitch/as_to_m, est_flux,0.,0.,0.]
model.set_model_params(model_bsf)
# Fine tune major position with BO
params_init = [(-.5+model_bsf[0],.5+model_bsf[0]),(-.5+model_bsf[1],.5+model_bsf[1])]
res = gbrt_minimize(model.model_err_major_pos, params_init)
model_bsf[0] = res.x[0]
model_bsf[1] = res.x[1]
model.set_pos_a(model_bsf[0],model_bsf[1])
# Tune major source flux with BO
res = gp_minimize(model.model_err_major_flux, [(0.2,2.)])
model.set_major_flux(res.x[0])
model_bsf[2]=res.x[0]
# Step 3: minor source position
residual = detector_image - model.model_image()
est_flux = residual.max(axis=None)*1. / src_peak_photons
model.set_minor_flux(est_flux)
model_bsf[5]=est_flux
# Use cross correlation to estimate minor source location
corr = signal.correlate2d(residual, psf_template, boundary='symm', mode='same')
max_loc = peak_local_max(corr,num_peaks=1)[0]
model_bsf[3] = (max_loc[0] - 256)*detector_pitch/as_to_m
model_bsf[4] = (max_loc[1] - 256)*detector_pitch/as_to_m
# Fine tune minor position with BO
res = gbrt_minimize(model.model_err_minor_pos, [(-.5+model_bsf[3],.5+model_bsf[3]),(-.5+model_bsf[4],.5+model_bsf[4])])
model_bsf[3] = res.x[0]
model_bsf[4] = res.x[1]
model.set_pos_b(res.x[0],res.x[1])
# Final tweak
params_init = [ (-.1+model_bsf[0],.1+model_bsf[0]),
(-.1+model_bsf[1],.1+model_bsf[1]),
(-.2+model_bsf[2],.2+model_bsf[2]),
(-.1+model_bsf[3],.1+model_bsf[3]),
(-.1+model_bsf[4],.1+model_bsf[4]),
(-.2+model_bsf[5],.2+model_bsf[5])]
res = gbrt_minimize(model.model_err, params_init, x0=[model_bsf])
return res
def opt_content_based():
space = [
Integer(0, 100), # artist
Integer(0, 100), # album
Integer(0, 100), # inferred album
Integer(0, 100), # duration_tracks
Integer(0, 100), # inferred duration
Integer(0, 100), # playcount
Integer(0, 100), # inferred playcount
Integer(0, 100), # tags
Integer(0, 100), # n_rating
Integer(0, 100), # urm weight
]
x0 = [100, 90, 80, 20, 10, 20, 10, 10, 10, 50]
res = gbrt_minimize(objective,
space,
x0=x0,
verbose=True,
n_random_starts=20,
n_calls=10000,
n_jobs=-1,
callback=result)
print('Maximimum p@k found: {:6.5f}'.format(-res.fun))
print('Optimal parameters:')
params = [
'Artist', 'Album', 'inferred_album', 'duration_tracks',
'inferred_duration', 'playcount', 'inferred_playcount', 'tags',
'n_rating', 'urm_weight'
]
for (p, x_) in zip(params, res.x):
print('{}: {}'.format(p, x_))
def minimize(self, initial_shape, bounds, c3, n_random_starts, n_calls):
return gbrt_minimize(self.change_shape_and_calculate,
dimensions=bounds,
x0=initial_shape,
y0=c3,
n_random_starts=n_random_starts,
n_calls=n_calls)
def tune():
tune_results = gbrt_minimize(
func=layout_cost,
dimensions=PARAMS_SPACE,
# callback=skopt.callbacks.DeltaYStopper(delta=5e-4, n_best=4),
n_calls=20,
)
best_geometry = params_to_geometry(tune_results.x)
return best_geometry, tune_results
def _suggest_x(dims, x0, y0, random_start, random_state, opts):
res = skopt.gbrt_minimize(lambda *args: 0,
dims,
n_calls=1,
n_random_starts=1 if random_start else 0,
x0=x0,
y0=y0,
random_state=random_state,
kappa=opts["kappa"],
xi=opts["xi"])
return res.x_iters[-1], res.random_state
def fit(self, X, Y, wavelengths):
def cost_func(guess):
assert(np.all(np.isfinite(guess))), guess
guess = dict(zip(self.opt_vars, guess))
return np.nanmedian(np.abs((self(X, wavelengths, **guess) - Y) / Y))
return np.abs(np.nanmean(self(X, wavelengths, **guess) - Y))
return ((self(X, wavelengths, **guess) - Y) ** 2).sum() ** 0.5
from skopt import gbrt_minimize
init = [(1e-2,100)]*len(self.opt_vars)
# res = minimize(cost_func, init, tol=1e-6, options={'maxiter':1e3}, method='BFGS')
res = gbrt_minimize(cost_func, init, n_random_starts=10000, n_calls=10000)#, method='SLSQP')#, tol=1e-10, options={'maxiter':1e5}, method='SLSQP')
print(self.__name__, res.x, res.fun)
self.trained_function = partial(self.function, wavelengths=wavelengths, **dict(zip(self.opt_vars, res.x)))
def _init_trial(trial, state):
import skopt
random_starts, x0, y0, dims = state.minimize_inputs(trial.run_id)
res = skopt.gbrt_minimize(
lambda *args: 0,
dims,
n_calls=1,
n_random_starts=random_starts,
x0=x0,
y0=y0,
random_state=state.random_state,
kappa=state.batch_flags["kappa"],
xi=state.batch_flags["xi"])
state.random_state = res.random_state
return skopt_util.trial_flags(state.flag_names, res.x_iters[-1])
def run_session(self):
super(SKOptSession, self)._write_headers(path=self.RESULTS_PATH)
for planner in self.planners:
params_set = dict(self.planner_config[planner].items())
search_space = self._load_search_space(params_set)
self.n_trial = 0 # Reset to n_trials to zero for each planner
self.planner = planner # Keeping track of current planner
self.start_time = timer() # Keeping track of start_time
if (self.MAX_RUNTIME != 'None'):
self.end_time = self.start_time + self.MAX_RUNTIME # Keeping track of end_time
rospy.loginfo('Executing %s on %s for %d secs', self.MODE,
planner, self.MAX_RUNTIME)
else:
rospy.loginfo('Executing %s on %s for %d trials', self.MODE,
planner, self.MAX_TRIALS)
if self.MODE == 'gp':
result = gp_minimize(self._skopt_obj,
search_space,
n_calls=self.MAX_TRIALS,
random_state=0,
acq_func='gp_hedge')
# gp_hedge means probabilistically choose betwn LCB, EI and PI acquisition functions at every iteration
elif self.MODE == 'rf':
result = forest_minimize(self._skopt_obj,
search_space,
n_calls=self.MAX_TRIALS,
random_state=0,
base_estimator='RF',
acq_func='EI')
elif self.MODE == 'et':
result = forest_minimize(self._skopt_obj,
search_space,
n_calls=self.MAX_TRIALS,
random_state=0,
base_estimator='ET',
acq_func='EI')
elif self.MODE == 'gbrt':
result = gbrt_minimize(self._skopt_obj,
search_space,
n_calls=self.MAX_TRIALS,
random_state=0,
acq_func='EI')
rospy.loginfo('Saved results to %s\n', self.RESULTS_PATH)
def opt_mf():
# Parameter to optimize:
# components, user_reg, item_reg, l_rate
space = [Integer(1, 10), # components
Real(1e-9, 0.8), # user_reg
Real(1e-9, 0.8), # pos_item_reg
Real(1e-9, 0.8), # neg_item_reg
Real(1e-5, 1), # l_rate
]
x0 = [2, 1e-2, 1e-3, 1e-3, 5e-3]
x1 = [3, 1e-2, 1e-3, 1e-2, 5e-3]
x2 = [4, 1e-2, 1e-3, 1e-2, 5e-1]
x3 = [4, 1e-2, 1e-2, 5e-3, 1e-1]
x4 = [5, 1e-2, 1e-3, 1e-3, 5e-2]
x0s = [x0, x1, x2, x3, x4]
# get the current fold
res = gbrt_minimize(objective, space, x0=x0s, verbose=True, n_random_starts=20, n_calls=1000, xi=0.01,n_jobs=-1, callback=result)
print('Maximimum p@k found: {:6.5f}'.format(-res.fun))
print('Optimal parameters:')
params = ['components', 'user_reg', 'CSLIM']
for (p, x_) in zip(params, res.x):
print('{}: {}'.format(p, x_))
def globalVQE(ham_dict,
TN,
initial_guess=None,
n_calls=100,
pre_applied_circ=None):
"""Optimizes the TN by simultaneously changing all the parameters
"""
q = QuantumRegister(TN.n_qubits)
c = ClassicalRegister(TN.n_qubits)
def objective(x):
TN.params = x
return measure_ham(q,
c,
ham_dict,
TN,
pre_applied_circ=pre_applied_circ)
res = gbrt_minimize(objective, [(0, 2 * pi)] * TN.n_params,
n_calls=n_calls,
verbose=False,
n_random_starts=30)
return res.fun
def optimize(X,
y,
range_file,
learner_choice='brtR',
optimizer='gp',
cv_folds=10,
n_calls=50):
# read in ranges from range_file
ranges = read_csv(range_file)
# get number of features of input data
n_features = X.shape[1]
# make dataframe of limits parse-able by objective
space = space_maker(ranges)
# make weights
weights = bernoulli(list(y))
# make compatible search spaces and objective functions for each learner_choice
if (learner_choice == 'brtR' or learner_choice == 'brtC'):
space += [(1, n_features)]
# define objective function to minimize <- skopt
def objective(params):
(learning_rate, max_depth, n_estimators, max_features) = params
if (learner_choice == 'brtR'):
learner = GradientBoostingRegressor()
elif (learner_choice == 'brtC'):
learner = GradientBoostingClassifier()
learner.set_params(learning_rate=learning_rate,
loss='ls',
max_depth=max_depth,
max_features=max_features,
n_estimators=n_estimators,
subsample=0.75)
return -mean(
cross_val_score(learner,
X,
y,
cv=cv_folds,
n_jobs=-1,
scoring="neg_mean_absolute_error"))
elif (learner_choice == 'xgbR' or learner_choice == 'xgbC'):
# define objective function to minimize <- xgboost
def objective(params):
(learning_rate, max_depth, n_estimators) = params
if (learner_choice == 'xgbR'):
learner = XGBRegressor()
elif (learner_choice == 'xgbC'):
learner = XGBClassifier()
learner.set_params(booster='gbtree',
learning_rate=learning_rate,
max_depth=max_depth,
n_estimators=n_estimators,
subsample=0.75)
return -mean(
cross_val_score(learner,
X,
y,
cv=cv_folds,
n_jobs=-1,
scoring="neg_mean_absolute_error"))
else:
error_lrn = 'not an available choice, use: brtR, brtC, xgbR, or xgbC'
# run fits: gaussian process, random forest, gradient-boosted regression trees (resp.)
if (optimizer == 'gp'):
best_pars = skopt.gp_minimize(objective, space, n_calls=n_calls)
elif (optimizer == 'rf'):
best_pars = skopt.forest_minimize(objective, space, n_calls=n_calls)
elif (optimizer == 'brt'):
best_pars = skopt.gbrt_minimize(objective, space, n_calls=n_calls)
else:
error_opt = 'not an available choice, use: gp, rf, or brt'
# return fit params conditional on no user type-o's
if 'error_opt' in locals():
print(error_opt)
return None
elif 'error_lrn' in locals():
print(error_lrn)
return None
else:
return best_pars.x
from matplotlib.colors import LogNorm
from skopt import gbrt_minimize
from skopt.benchmarks import branin
from skopt.acquisition import gaussian_ei
plt.figure(figsize=(10, 10))
plt.set_cmap("viridis")
dimensions = [(-5.0, 10.0), (0.0, 15.0)]
x1_values = np.linspace(-5, 10, 100)
x2_values = np.linspace(0, 15, 100)
x_ax, y_ax = np.meshgrid(x1_values, x2_values)
vals = np.c_[x_ax.ravel(), y_ax.ravel()]
res = gbrt_minimize(
branin, dimensions, maxiter=200, random_state=1)
model = res.models[-1]
opt_points = res.x_iters
y_opt = res.fun
x_opt = res.x
acquis_values = gaussian_ei(vals, model, y_opt)
acquis_values = acquis_values.reshape(100, 100)
branin_vals = np.reshape([branin(val) for val in vals], (100, 100))
plt.subplot(211)
plt.pcolormesh(x_ax, y_ax, acquis_values)
plt.plot(opt_points[:, 0], opt_points[:, 1], 'ro',
markersize=4, lw=0, label='samples')
callback=None,
n_points=num,
xi=0.01,
kappa=1.96,
n_jobs=1)
elif method == 'GradientBoostTree':
gbrt_minimize(simulator.objfunction,
dimensions=dimensions,
base_estimator=None,
n_calls=num,
n_random_starts=10,
acq_func='EI',
acq_optimizer='auto',
x0=None,
y0=None,
random_state=None,
verbose=False,
callback=None,
n_points=num,
xi=0.01,
kappa=1.96,
n_jobs=1)
elif method == 'GaussianProcess':
gp_minimize(simulator.objfunction,
dimensions=dimensions,
base_estimator=None,
n_calls=num,
n_random_starts=10,
import matplotlib.pyplot as plt
from skopt import gbrt_minimize
from skopt.benchmarks import bench3
from skopt.gbrt_opt import _expected_improvement
bounds = [[-1, 1]]
x = np.linspace(-1, 1, 200)
func_values = [bench3(xi) for xi in x]
plt.figure(figsize=(10, 5))
vals = np.reshape(x, (-1, 1))
col_no = 1
row_no = 6
res = gbrt_minimize(
bench3, bounds, maxiter=6, n_start=1, random_state=1)
best_xs = res.x_iters.ravel()
best_ys = res.func_vals.ravel()
models = res.models
for n_iter in range(5):
model = models[n_iter]
best_x = best_xs[:n_iter+1]
best_y = best_ys[:n_iter+1]
low, mu, high = model.predict(vals).T
std = (high - low) / 2
acquis_values = _expected_improvement(vals, model, best_y[-1])
acquis_values = acquis_values.ravel()
posterior_mean = mu.ravel()
posterior_std = std.ravel()
def findbestparams(modelclass,
encoder,
corpus,
modelsfolder,
n_calls=100,
verbose=1,
valmask=None,
patience=20,
maxepochs=1000,
checkpointfile=None):
"""Find the best parameters for a given model architecture and param grid
Returns
- list with the best parameters found for the model
- validation loss for such model
- best model found
- OptimizeResult object with info on the optimization procedure
"""
if verbose >= 1:
print("Will save all tested models under %s" % modelsfolder)
# Load checkpoint (if any)
x0 = None
y0 = None
previoustrials = 0
if checkpointfile:
prevtrials = checkpointload(checkpointfile)
if len(prevtrials) > 0:
params, losses = zip(*prevtrials)
x0 = list(params)
y0 = list(losses)
previoustrials = len(x0)
if verbose >= 1:
print("Checkpoint x0", x0)
print("Checkpoint y0", y0)
# Prepare and run optimizer
fobj = createobjective(modelclass,
encoder,
corpus,
verbose=verbose,
valmask=valmask,
patience=patience,
maxepochs=maxepochs,
modelsfolder=modelsfolder,
checkpointfile=checkpointfile)
grid = addoptimizerparams(modelclass.paramgrid)
trials = max(n_calls - previoustrials,
1) # Run remaining trials, minimum 10
randomtrials = max(RANDOMTRIALS - previoustrials, 0)
optres = gbrt_minimize(fobj,
grid,
n_calls=trials,
n_random_starts=randomtrials,
random_state=0,
x0=x0,
y0=y0)
# Recover best parameters, best loss, best model
bestparams = optres.x
bestloss = optres.fun
bestmodel = load_model(modelsfolder + "/" + loss2modelname(bestloss))
return bestparams, bestloss, bestmodel, optres
def main(config):
"""Main Function"""
seed = np.random.randint(0, high=10000)
if 'seed' in config:
seed = config['seed']
torch.manual_seed(seed)
np.random.seed(seed)
protected_index = descriptions.index(config['protected_attr'])
prediction_index = descriptions.index(config['prediction_attr'])
valid_results, test_results = {}, {}
_, _, _, trainloader, valloader, testloader = load_celeba(
trainsize=config['trainsize'],
testsize=config['testsize'],
num_workers=config['num_workers'],
batch_size=config['batch_size'])
if config['print_priors']:
logger.info('train priors')
compute_priors(trainloader, protected_index, prediction_index)
logger.info('val priors')
compute_priors(valloader, protected_index, prediction_index)
logger.info('test priors')
compute_priors(testloader, protected_index, prediction_index)
net = get_resnet_model()
criterion = nn.BCEWithLogitsLoss()
if config['optimizer'] == 'sgd':
optimizer = optim.SGD(net.parameters(), lr=config['lr'])
else:
optimizer = optim.Adam(net.parameters(), lr=config['lr'])
checkpoint_file = Path('seed' + str(seed) + config['optimizer'] +
str(config['lr']) + '_pro_' +
config['protected_attr'] + '_pre_' +
config['prediction_attr'] + '_' +
config['checkpoint'])
start_epoch = 0
if checkpoint_file.is_file() and (not config['retrain']):
checkpoint = torch.load(checkpoint_file, map_location=device)
logger.info('loaded from %s', checkpoint_file)
net.load_state_dict(checkpoint['model_state_dict'])
optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
start_epoch = checkpoint['epoch']
else:
train_model(net,
trainloader,
valloader,
criterion,
optimizer,
str(checkpoint_file),
protected_index,
prediction_index,
epochs=config['epochs'],
start_epoch=start_epoch)
_, best_thresh = val_model(net, valloader, get_best_balanced_accuracy,
protected_index, prediction_index)
logger.info(f'val_results, thresh, {best_thresh.item()}')
valid_results['base_model'] = print_objective_results(
valloader, net, best_thresh, protected_index, prediction_index)
logger.info('test_results')
result_dict = print_objective_results(testloader, net, best_thresh,
protected_index, prediction_index)
test_results['base_model'] = result_dict
def to_dataframe(y_true, y_pred, y_prot):
y_true, y_pred, y_prot = y_true.float().cpu().numpy(), y_pred.float(
).cpu().numpy(), y_prot.float().cpu().numpy()
df = pd.DataFrame({
'y_true': y_true,
'y_pred': y_pred,
'y_prot': y_prot
})
dataset = StandardDataset(df, 'y_true', [1.], ['y_prot'], [[1.]])
dataset.scores = y_pred.reshape(-1, 1)
return dataset
val_dataset = val_model(net, valloader, to_dataframe, protected_index,
prediction_index)
test_dataset = val_model(net, testloader, to_dataframe, protected_index,
prediction_index)
def eval_aif360_algorithm(y_pred, dataset, verbose=True):
global yaml_config
acc = float(np.mean(y_pred == dataset.labels.reshape(-1)))
bias = compute_bias(
torch.tensor(y_pred), torch.tensor(dataset.labels.reshape(-1)),
torch.tensor(dataset.protected_attributes.reshape(-1)),
yaml_config['metric']).item()
obj = compute_objective(acc, bias, margin=0.0)
if verbose:
logger.info(f'accuracy {acc}')
logger.info(f'{yaml_config["metric"]} {bias}')
logger.info(f'objective {obj}')
return {
'roc_auc': None,
'accuracy': float(acc),
'bias': float(bias),
'objective': float(obj)
}
# Evaluate ROC
if "ROC" in config['models']:
ROC = RejectOptionClassification(unprivileged_groups=[{
'y_prot': 1.
}],
privileged_groups=[{
'y_prot': 0.
}],
low_class_thresh=0.01,
high_class_thresh=0.99,
num_class_thresh=100,
num_ROC_margin=50,
metric_name="Average odds difference",
metric_ub=0.05,
metric_lb=-0.05)
logger.info("Training ROC model with validation dataset.")
ROC = ROC.fit(val_dataset, val_dataset)
logger.info("Evaluating ROC model.")
logger.info('ROC val results')
val_y_pred = ROC.predict(val_dataset).labels.reshape(-1)
valid_results['ROC'] = eval_aif360_algorithm(val_y_pred, val_dataset)
logger.info('ROC test results')
test_y_pred = ROC.predict(test_dataset).labels.reshape(-1)
test_results['ROC'] = eval_aif360_algorithm(test_y_pred, test_dataset)
ROC = None
if 'EqOdds' in config['models']:
eo = EqOddsPostprocessing(privileged_groups=[{
'y_prot': 0.
}],
unprivileged_groups=[{
'y_prot': 1.
}])
logger.info("Training Equality of Odds model with validation dataset.")
eo = eo.fit(val_dataset, val_dataset)
logger.info("Evaluating Equality of Odds model.")
logger.info('Equality of Odds val results')
val_y_pred = eo.predict(val_dataset).labels.reshape(-1)
valid_results['EqOdds'] = eval_aif360_algorithm(
val_y_pred, val_dataset)
logger.info('Equality of Odds test results')
test_y_pred = eo.predict(test_dataset).labels.reshape(-1)
test_results['EqOdds'] = eval_aif360_algorithm(test_y_pred,
test_dataset)
eo = None
if 'CalibEqOdds' in config['models']:
cost_constraint = config['CalibEqOdds']['cost_constraint']
cpp = CalibratedEqOddsPostprocessing(privileged_groups=[{
'y_prot': 0.
}],
unprivileged_groups=[{
'y_prot': 1.
}],
cost_constraint=cost_constraint)
logger.info(
"Training Calibrated Equality of Odds model with validation dataset."
)
cpp = cpp.fit(val_dataset, val_dataset)
logger.info("Evaluating Calibrated Equality of Odds model.")
logger.info('Calibrated Equality of Odds val results')
valid_y_pred = cpp.predict(val_dataset).labels.reshape(-1)
valid_results['CalibEqOdds'] = eval_aif360_algorithm(
valid_y_pred, val_dataset)
logger.info('Equality of Odds test results')
test_y_pred = cpp.predict(test_dataset).labels.reshape(-1)
test_results['CalibEqOdds'] = eval_aif360_algorithm(
valid_y_pred, val_dataset)
cpp = None
if 'random' in config['models']:
rand_result = [-np.inf, None, -1]
for iteration in range(101):
rand_model = copy.deepcopy(net)
rand_model.to(device)
for param in rand_model.parameters():
param.data = param.data * (torch.randn_like(param) * 0.1 + 1)
rand_model.eval()
best_obj, best_thresh = val_model(rand_model, valloader,
get_best_objective,
protected_index,
prediction_index)
logger.info(f'iteration {iteration} obj {float(best_obj)}')
if best_obj > rand_result[0]:
logger.info('found new best')
del rand_result[1]
rand_result = [
best_obj,
copy.deepcopy(rand_model.state_dict()), best_thresh
]
if iteration % 10 == 0:
logger.info(f"{iteration} / 101 trials have been sampled.")
logger.info(f'current best obj {float(rand_result[0])}')
# evaluate best random model
best_model = copy.deepcopy(net)
best_model.load_state_dict(rand_result[1])
best_model.to(device)
best_thresh = rand_result[2]
logger.info('val_results')
valid_results['random'] = print_objective_results(
valloader, best_model, best_thresh, protected_index,
prediction_index)
logger.info('test_results')
test_results['random'] = print_objective_results(
testloader, best_model, best_thresh, protected_index,
prediction_index)
torch.save(
best_model.state_dict(),
'seed' + str(seed) + config['metric'] + config['optimizer'] +
str(config['lr']) + '_pro_' + config['protected_attr'] + '_pre_' +
config['prediction_attr'] + config['random']['checkpoint'])
if 'layerwiseOpt' in config['models']:
base_model = copy.deepcopy(net)
best_state_dict, best_thresh, best_obj = None, None, np.inf
total_params = len(list(base_model.parameters()))
for index, param in enumerate(base_model.parameters()):
if index < total_params - config['layerwiseOpt']['num_layers']:
continue
logger.info(f'Evaluating param number {index} of {total_params}')
param_copy = copy.deepcopy(param)
def objective(new_param):
param.data[indices] = torch.tensor(new_param).to(device)
base_model.eval()
best_obj, thresh = val_model(base_model, valloader,
get_best_objective,
protected_index, prediction_index)
logger.info(
f'Evaluating param number {index} of {total_params}')
return -float(best_obj)
std = param.flatten().cpu().detach().numpy().std()
num_elems = param.size().numel()
ratio = min(1., config['layerwiseOpt']['max_sparsity'] / num_elems)
indices = torch.rand(param.size()) < ratio
space = [
Real(
float(x.cpu().detach()) - 2.2 * std,
float(x.cpu().detach()) + 2.2 * std)
for x in param[indices]
]
logger.info(f'Number of sparse indices: {indices.sum().item()}')
res_gbrt = gbrt_minimize(objective,
space,
n_calls=20,
verbose=True)
if res_gbrt.fun < best_obj:
param.data[indices] = torch.tensor(res_gbrt.x).to(device)
best_state_dict = copy.deepcopy(base_model.state_dict())
best_obj, best_thresh = val_model(base_model, valloader,
get_best_objective,
protected_index,
prediction_index)
param.data = param_copy.data
best_model = copy.deepcopy(net)
best_model.load_state_dict(best_state_dict)
best_model.to(device)
logger.info('val_results')
valid_results['layerwiseOpt'] = print_objective_results(
valloader, best_model, best_thresh, protected_index,
prediction_index)
logger.info('test_results')
test_results['layerwiseOpt'] = print_objective_results(
testloader, best_model, best_thresh, protected_index,
prediction_index)
if 'adversarial' in config['models']:
unrefined_net = get_resnet_model()
base_model = copy.deepcopy(unrefined_net)
base_model.fc = nn.Linear(base_model.fc.in_features,
base_model.fc.in_features)
actor = nn.Sequential(base_model,
nn.Linear(base_model.fc.in_features, 2))
actor.to(device)
actor_optimizer = optim.Adam(actor.parameters(), lr=1e-4)
actor_loss_fn = nn.BCEWithLogitsLoss()
actor_loss = 0.
actor_steps = config['adversarial']['actor_steps']
critic = Critic(config['batch_size'] * unrefined_net.fc.in_features)
critic.to(device)
critic_optimizer = optim.Adam(critic.parameters(), lr=1e-4)
critic_loss_fn = nn.MSELoss()
critic_loss = 0.
critic_steps = config['adversarial']['critic_steps']
for epoch in range(config['adversarial']['epochs']):
for param in critic.parameters():
param.requires_grad = True
for param in actor.parameters():
param.requires_grad = False
actor.eval()
critic.train()
for step, (inputs, labels) in enumerate(valloader):
if step > critic_steps:
break
inputs, labels = inputs.to(device), labels.to(device)
if inputs.size(0) != config['batch_size']:
continue
critic_optimizer.zero_grad()
with torch.no_grad():
y_pred = actor(inputs)
y_true = labels[:, prediction_index].float().to(device)
y_prot = labels[:, protected_index].float().to(device)
bias = compute_bias(y_pred, y_true, y_prot, config['metric'])
res = critic(base_model(inputs))
loss = critic_loss_fn(bias.unsqueeze(0), res[0])
loss.backward()
critic_loss += loss.item()
critic_optimizer.step()
if step % 100 == 0:
print_loss = critic_loss if (
epoch * critic_steps + step
) == 0 else critic_loss / (epoch * critic_steps + step)
logger.info(
f'=======> Epoch: {(epoch, step)} Critic loss: {print_loss:.3f}'
)
for param in critic.parameters():
param.requires_grad = False
for param in actor.parameters():
param.requires_grad = True
actor.train()
critic.eval()
for step, (inputs, labels) in enumerate(valloader):
if step > actor_steps:
break
inputs, labels = inputs.to(device), labels.to(device)
if inputs.size(0) != config['batch_size']:
continue
actor_optimizer.zero_grad()
y_true = labels[:, prediction_index].float().to(device)
y_prot = labels[:, protected_index].float().to(device)
est_bias = critic(base_model(inputs))
loss = actor_loss_fn(actor(inputs)[:, 0], y_true)
loss = max(
1, config['adversarial']['lambda'] *
(abs(est_bias) - config['objective']['epsilon'] +
config['adversarial']['margin']) + 1) * loss
loss.backward()
actor_loss += loss.item()
actor_optimizer.step()
if step % 100 == 0:
print_loss = critic_loss if (
epoch * actor_steps + step
) == 0 else critic_loss / (epoch * actor_steps + step)
logger.info(
f'=======> Epoch: {(epoch, step)} Actor loss: {print_loss:.3f}'
)
print_objective_results(valloader, actor, best_thresh,
protected_index, prediction_index)
_, best_thresh = val_model(actor, valloader, get_best_objective,
protected_index, prediction_index)
logger.info('val_results')
valid_results['adversarial'] = print_objective_results(
valloader, actor, best_thresh, protected_index, prediction_index)
logger.info('test_results')
test_results['adversarial'] = print_objective_results(
testloader, actor, best_thresh, protected_index, prediction_index)
torch.save(actor.state_dict(), config['adversarial']['checkpoint'])
with open(
'valid_seed' + str(seed) + config['metric'] + config['optimizer'] +
str(config['lr']) + '_pro_' + config['protected_attr'] + '_pre_' +
config['prediction_attr'] + config['output'], 'w') as filehandler:
json.dump(valid_results, filehandler)
with open(
'test_seed' + str(seed) + config['metric'] + config['optimizer'] +
str(config['lr']) + '_pro_' + config['protected_attr'] + '_pre_' +
config['prediction_attr'] + config['output'], 'w') as filehandler:
json.dump(test_results, filehandler)
def train():
(xtrain, ytrain), (xvalid, yvalid), (_, _), y_std, y_mean = get_mc_data(FLAGS.dataset_name)
min_tau = setup['tau_range'][FLAGS.dataset_name][0]
max_tau = setup['tau_range'][FLAGS.dataset_name][1]
# FIND BEST N_EPOCHS; TAU
with tf.Graph().as_default() as g:
with tf.Session() as sess:
saver, model_dir, pyx, x, n_epochs = train_model(sess, xtrain, ytrain, FLAGS.seed, FLAGS.seed, xvalid, yvalid)
# CRPS OPTIMIZE TO FIND STD DEV
print "Finding optimal Tau"
saver.restore(sess, model_dir + 'model')
# Make optimization run take extra arguments
optimize_fun = partial(run_tau_opt, sess, pyx, x, y_std, y_mean, xvalid, yvalid, False)
tau_opt = gbrt_minimize(optimize_fun,
[(min_tau, max_tau)],
n_random_starts=100,
n_calls=200
)
opt_tau = tau_opt.x[0]
# CRPS OPTIMIZE TO FIND STD DEV
print "Finding optimal CU Tau"
# Make optimization run take extra arguments
optimize_fun = partial(run_tau_opt, sess, pyx, x, y_std, y_mean, xvalid, yvalid, True)
cutau_opt = gbrt_minimize(optimize_fun,
[(min_tau, max_tau)],
n_random_starts=100,
n_calls=200
)
cu_opt_tau = cutau_opt.x[0]
print "OPT TAU: {}. CRPS: {}".format(opt_tau, tau_opt.fun)
print "CU OPT TAU: {}. CRPS: {}".format(cu_opt_tau, cutau_opt.fun)
tf.reset_default_graph()
# TRAIN AND EVALUATE FINAL MODEL 5 TIMES WITH DIFFERENT SEED:
for final_seed in range(FLAGS.seed + 1, FLAGS.seed + 6):
(xtrain, ytrain), (xtest, ytest), y_std, y_mean = get_mc_data(FLAGS.dataset_name, False)
with tf.Graph().as_default() as g:
with tf.Session() as sess:
# Write csv file column headers if not yet written.
plot_file_path = os.path.join(PLOT_RESULTS_PATH, FLAGS.dataset_name + '.txt')
fid = open(plot_file_path, 'a')
if sum(1 for line in open(plot_file_path)) == 0:
fid.write(',MNF const std dev,MNF std dev,y,yHat,run_count,dataset_split_seed\n')
pll_file_path = os.path.join(RESULTS_PATH, FLAGS.dataset_name + '-pll.txt')
fid_pll = open(pll_file_path, 'a')
if sum(1 for line in open(pll_file_path)) == 0:
fid_pll.write('dataset_split,run_count,pll result,pll baseline,pll best,pll normalized\n')
crps_file_path = os.path.join(RESULTS_PATH, FLAGS.dataset_name + '-crps.txt')
fid_crps = open(crps_file_path, 'a')
if sum(1 for line in open(crps_file_path)) == 0:
fid_crps.write('dataset_split,run_count,crps result,crps baseline,crps best,crps normalized\n')
rmse_file_path = os.path.join(RESULTS_PATH, FLAGS.dataset_name + '-rmse.txt')
fid_rmse = open(rmse_file_path, 'a')
if sum(1 for line in open(rmse_file_path)) == 0:
fid_rmse.write('dataset_split,run_count,rmse\n')
saver, model_dir, pyx, x, _ = train_model(sess, xtrain, ytrain, final_seed, final_seed, xvalid=False, yvalid=False, n_epochs=n_epochs)
#EVALUATE TEST SET
preds = np.zeros_like(ytest)
all_preds = np.zeros([len(ytest), FLAGS.L])
widgets = ["Sampling |", Percentage(), Bar(), ETA()]
pbar = ProgressBar(FLAGS.L, widgets=widgets)
pbar.start()
for i in range(FLAGS.L):
pbar.update(i)
for j in range(int(xtest.shape[0] / 100)):
pyxi = sess.run(pyx, feed_dict={x: xtest[j * 100:(j + 1) * 100]})
preds[j * 100:(j + 1) * 100] += pyxi / FLAGS.L
all_preds[j * 100:(j + 1) * 100, i] = np.squeeze(pyxi * y_std + y_mean)
pyxi = sess.run(pyx, feed_dict={x: xtest[int(xtest.shape[0] / 100) * 100:]})
preds[int(xtest.shape[0] / 100) * 100:] += pyxi / FLAGS.L
all_preds[int(xtest.shape[0] / 100) * 100:, i] = np.squeeze(pyxi * y_std + y_mean)
# FIND PLL AND CRPS
samples = all_preds[:, :, newaxis].T.reshape(FLAGS.L, len(all_preds), 1)
mean, var = np.mean(samples, axis=0), np.var(samples, axis=0) + opt_tau ** (-1)
pll_res = pll(samples, ytest * y_std + y_mean, FLAGS.L, opt_tau)
crps_res = crps(ytest * y_std + y_mean, mean, var)
# FIND BASELINE PLL AND CRPS
pll_baseline = pll(np.array([mean]), ytest * y_std + y_mean, 1, cu_opt_tau)
crps_baseline = crps(ytest * y_std + y_mean, mean, cu_opt_tau**(-1))
# FIND OPTIMAL PLL AND CRPS
pll_best = pll_maximum(mean, ytest * y_std + y_mean)
crps_best = crps_minimum(mean, ytest * y_std + y_mean)
# GET NORMALIZED SCORES
pll_norm = (pll_res - pll_baseline) / (pll_best - pll_baseline)
crps_norm = (crps_res - crps_baseline) / (crps_best - crps_baseline)
sample_accuracy = np.sqrt(np.mean((preds-ytest)*(preds-ytest)))
print('Sample test accuracy: {}'.format(sample_accuracy))
ytest_u = (ytest * y_std + y_mean)
preds_u = (preds * y_std + y_mean)
unnormalized_rmse = np.sqrt(np.mean((preds_u - ytest_u) * (preds_u - ytest_u)))
print('Sample test accuracy (unnormalized): {}'.format(unnormalized_rmse))
print('Test uncertainty quality metrics:')
print "PLL: {}, PLL LOWER: {}, PLL UPPER: {}, NORM: {}".format(pll_res, pll_baseline, pll_best, pll_norm)
print "CRPS: {}, CRPS LOWER: {}, CRPS UPPER: {}, NORM: {}".format(crps_res, crps_baseline, crps_best, crps_norm)
all_preds_mean = all_preds.mean(axis=1)
all_preds_std = all_preds.std(axis=1)
# Write results to files
for i in range(len(ytest)):
fid.write('%d,%f,%f,%f,%f,%d,%d\n' % (i, np.sqrt(cu_opt_tau**(-1)), all_preds_std[i], ytest_u[i], all_preds_mean[i], final_seed, FLAGS.dataset_split_seed))
fid_rmse.write('%d,%d,%f\n' % (FLAGS.dataset_split_seed, final_seed, unnormalized_rmse))
fid_pll.write('%d,%d %f,%f,%f,%f\n' % (FLAGS.dataset_split_seed, final_seed, pll_res, pll_baseline, pll_best, pll_norm))
fid_crps.write('%d,%d,%f,%f,%f,%f\n' % (FLAGS.dataset_split_seed, final_seed, crps_res, crps_baseline, crps_best, crps_norm))
fid.close()
fid_rmse.close()
fid_pll.close()
fid_crps.close()
tf.reset_default_graph()
print('Last epoch: ', epoch)
print('Credit Cost: ', CreditCost)
if os.path.exists('checkpoint.pt'):
os.remove('checkpoint.pt')
print()
torch.cuda.empty_cache()
print()
return CreditCost
# not working #res_gp = gp_minimize(objective, dimensions=dimensions, n_calls=TRIALS, random_state=1, verbose=True, acq_func='gp_hedge', acq_optimizer='auto', n_jobs=1)
# res_gp = forest_minimize(objective, dimensions=dimensions, base_estimator='RF', n_calls=TRIALS, n_random_starts=RANDOM_STARTS, acq_func='EI', x0=None, y0=None, random_state=None, verbose=True, callback=None, n_points=10000, xi=0.01, kappa=1.96, n_jobs=128)
res_gp = gbrt_minimize(objective, dimensions=dimensions, base_estimator='ET', n_calls=TRIALS + RANDOM_STARTS, n_random_starts=RANDOM_STARTS, acq_func='LCB', x0=None, y0=None, random_state=None, verbose=True, callback=None, n_points=100, xi=0.01, kappa=1.96, n_jobs=1)
# res_gp = dummy_minimize(objective, dimensions=dimensions, n_calls=TRIALS, x0=None, y0=None, random_state=None, verbose=True, callback=None)
"Best score=%.4f" % res_gp.fun
print("""Best parameters: - optimization=%d""" % (res_gp.x[0]))
print(res_gp)
plot_convergence(res_gp)
# plot_evaluations(res_gp)
# plot_objective(res_gp)
# In[4]:
if OPTIMIZATION_PLUGIN == 'Bayesian' :
from bayes_opt import BayesianOptimization
from skopt import gbrt_minimize
from skopt.benchmarks import branin
from skopt.gbrt_opt import _expected_improvement
plt.figure(figsize=(10, 10))
plt.set_cmap("viridis")
bounds = np.asarray([[-5, 10], [0, 15]])
x1_values = np.linspace(-5, 10, 100)
x2_values = np.linspace(0, 15, 100)
x_ax, y_ax = np.meshgrid(x1_values, x2_values)
vals = np.c_[x_ax.ravel(), y_ax.ravel()]
res = gbrt_minimize(
branin, bounds, maxiter=200, random_state=1)
model = res.models[-1]
opt_points = res.x_iters
y_opt = res.fun
x_opt = res.x
acquis_values = _expected_improvement(vals, model, y_opt)
acquis_values = acquis_values.reshape(100, 100)
branin_vals = np.reshape([branin(val) for val in vals], (100, 100))
plt.subplot(211)
plt.pcolormesh(x_ax, y_ax, acquis_values)
plt.plot(opt_points[:, 0], opt_points[:, 1], 'ro',
markersize=4, lw=0, label='samples')
def layerwiseOpt_debiasing(model_state_dict, data, config, device):
logger.info('Training layerwiseOpt model.')
base_model = load_model(data.num_features,
config.get('hyperparameters', {}))
base_model.load_state_dict(model_state_dict)
base_model.to(device)
best_state_dict, best_obj, best_thresh = None, math.inf, -1
total_params = len(list(base_model.parameters()))
for index, param in enumerate(base_model.parameters()):
if index < total_params - config['layerwiseOpt']['num_layers']:
continue
logger.info(f'Evaluating param number {index} of {total_params}')
param_copy = copy.deepcopy(param)
def objective(new_param, return_thresh=False):
param.data[indices] = torch.tensor(new_param)
base_model.eval()
with torch.no_grad():
scores = base_model(data.X_valid_gpu)[:, 0].reshape(-1).numpy()
best_thresh, best_obj = get_best_thresh(
scores,
np.linspace(0, 1, 501),
data,
config,
valid=False,
margin=config['layerwiseOpt']['margin'])
print(f'Evaluating param number {index} of {total_params}')
if return_thresh:
return -float(best_obj), float(best_thresh)
return -float(best_obj)
mean = param.flatten().cpu().detach().numpy().mean()
std = param.flatten().cpu().detach().numpy().std()
num_elems = param.size().numel()
ratio = min(1., config['layerwiseOpt']['max_sparsity'] / num_elems)
indices = torch.rand(param.size()) < ratio
space = [
Real(
float(x.cpu().detach()) - 2.2 * std,
float(x.cpu().detach()) + 2.2 * std) for x in param[indices]
]
# std = param.flatten().cpu().detach().numpy().std()
# num_elems = param.size().numel()
# ratio = min(1., config['layerwiseOpt']['max_sparsity'] / num_elems)
# indices = torch.rand(param.size()) < ratio
logger.info(f'Number of sparse indices: {indices.sum().item()}')
res_gbrt = gbrt_minimize(objective,
space,
n_calls=config['layerwiseOpt']['n_calls'],
verbose=True)
if res_gbrt.fun < best_obj:
param.data[indices] = torch.tensor(res_gbrt.x)
best_state_dict = base_model.state_dict()
best_obj, best_thresh = objective(res_gbrt.x, return_thresh=True)
best_obj = -best_obj
param.data = param_copy.data
best_model = load_model(data.num_features,
config.get('hyperparameters', {}))
best_model.to(device)
best_model.load_state_dict(best_state_dict)
best_model.eval()
with torch.no_grad():
y_pred = (best_model(data.X_valid_gpu)[:, 0] >
best_thresh).reshape(-1).numpy()
results_valid = get_valid_objective(y_pred, data, config)
best_model.eval()
with torch.no_grad():
y_pred = (best_model(data.X_test_gpu)[:, 0] >
best_thresh).reshape(-1).numpy()
results_test = get_test_objective(y_pred, data, config)
return results_valid, results_test
def get_params_SKopt(model, X, Y, space, cv_search, alg = 'catboost', cat_features = None, eval_dataset = None, UBM = False, opt_method =
'gbrt_minimize', verbose = True, multi = False, scoring = 'neg_mean_squared_error', n_best = 50, total_time = 7200):
"""The method performs parameters tuning of an algorithm using scikit-optimize library.
Parameters:
1.
2.
3. multi - boolean, is used when a multioutput algorithm is tuned
UPDATES:
1. In this current version, the support of the catboost algorithms is added
"""
if alg == 'catboost':
fitparam = { 'eval_set' : eval_dataset,
'use_best_model' : UBM,
'cat_features' : cat_features,
'early_stopping_rounds': 20 }
else:
fitparam = {}
@use_named_args(space)
def objective(**params):
model.set_params(**params)
return -np.mean(cross_val_score(model,
X, Y,
cv=cv_search,
scoring= scoring,
fit_params=fitparam))
if opt_method == 'gbrt_minimize':
HPO_PARAMS = {'n_calls':1000,
'n_random_starts':20,
'acq_func':'EI',}
reg_gp = gbrt_minimize(objective,
space,
n_jobs = -1,
verbose = verbose,
callback = [DeltaYStopper(delta = 0.01, n_best = 5), RepeatedMinStopper(n_best = n_best), DeadlineStopper(total_time = total_time)],
**HPO_PARAMS,
random_state = RANDOM_STATE)
elif opt_method == 'forest_minimize':
HPO_PARAMS = {'n_calls':1000,
'n_random_starts':20,
'acq_func':'EI',}
reg_gp = forest_minimize(objective,
space,
n_jobs = -1,
verbose = verbose,
callback = [RepeatedMinStopper(n_best = n_best), DeadlineStopper(total_time = total_time)],
**HPO_PARAMS,
random_state = RANDOM_STATE)
elif opt_method == 'gp_minimize':
HPO_PARAMS = {'n_calls':1000,
'n_random_starts':20,
'acq_func':'gp_hedge',}
reg_gp = gp_minimize(objective,
space,
n_jobs = -1,
verbose = verbose,
callback = [RepeatedMinStopper(n_best = n_best), DeadlineStopper(total_time = total_time)],
**HPO_PARAMS,
random_state = RANDOM_STATE)
TUNED_PARAMS = {}
for i, item in enumerate(space):
if multi:
TUNED_PARAMS[item.name.split('__')[1]] = reg_gp.x[i]
else:
TUNED_PARAMS[item.name] = reg_gp.x[i]
return [TUNED_PARAMS,reg_gp]
def compare_optimizers(num_instances=4, graph_size=15, n_calls=8, n_random_starts=2):
global pbar
pbar = None
# For progress bar.
pbar = tqdm(total=num_instances*n_calls*4)
instances = [Graph(graph_size) for _ in range(num_instances)]
# Percent of optimal score acheived by each algorithm.
dummy = []
decision_trees = []
gradient_boosted_trees = []
baynesian = []
# For each instance, run each algorithm.
for inst in instances:
# Scikit functions only take in parameters and want to minimize values.
# Create a wrapper function to format get_expectation.
sk_get_exp = lambda x: -1*get_expectation(x, inst)
opt = inst.optimal_score()[0]
# Dummy.
inst.clear_runs()
dummy_minimize(func=sk_get_exp,
dimensions=[(0,2*np.pi),(0,np.pi)],
n_calls=n_calls)
dummy.append(float(inst.currentScore) / opt)
# Decision Trees.
inst.clear_runs()
forest_minimize(func=sk_get_exp,
dimensions=[(0,2*np.pi),(0,np.pi)],
n_calls=n_calls,
n_random_starts=n_random_starts)
decision_trees.append(float(inst.currentScore) / opt)
# Gradient Boosted Decision Trees.
inst.clear_runs()
gbrt_minimize(func=sk_get_exp,
dimensions=[(0,2*np.pi),(0,np.pi)],
n_calls=n_calls,
n_random_starts=n_random_starts)
gradient_boosted_trees.append(float(inst.currentScore) / opt)
# Baynesian.
inst.clear_runs()
gp_minimize(func=sk_get_exp,
dimensions=[(0,2*np.pi),(0,np.pi)],
n_calls=n_calls,
n_random_starts=n_random_starts)
baynesian.append(float(inst.currentScore) / opt)
# Compare mean/stdev of % opt. achieved for each algorithm.
print("-- % of Optimal Achieved, Mean and Std. Dev --")
print("Random Sampling:\nMean: %s\nStd. Dev: %s" % (mean(dummy), stdev(dummy)))
print("Decision Trees:\nMean: %s\nStd. Dev: %s" % (mean(decision_trees), stdev(decision_trees)))
print("Gradient Boosted Decision Trees:\nMean: %s\nStd. Dev: %s" % (mean(gradient_boosted_trees), stdev(gradient_boosted_trees)))
print("Baynesian Optimization:\nMean: %s\nStd. Dev: %s" % (mean(baynesian), stdev(baynesian)))
alpha = params['alpha']
elif method is "skopt":
alpha = params[0]
score = mapk_similarity(alpha)
if method is "hyperopt":
print("INFO: iteration {} error {:.3f}".format(sim_objective.i, score))
return 1-score
partial_objective = lambda params: sim_objective(params,
method=method)
hp_params = {'alpha': hp.uniform('alpha', 0.01, 1.)}
method = "hyperopt"
sim_objective.i=0
hp_best = fmin(fn=partial_objective,
space=hp_params,
algo=tpe.suggest,
max_evals=100)
sk_params = [(0.01, 1, 'uniform')]
method = "skopt"
sk_best = gbrt_minimize(partial_objective,
sk_params,
n_calls=100,
random_state=0,
verbose=False,
n_jobs=-1)
def any_order_VQE(ham_dict, TN, tensor_order=None, n_calls=20, method="GP"):
"""
Performs VQE by optimizing the tensor network in the order
supplied in tensor_order. The n_sweeps parameter is redundant as
one can simply send a duplicate of the tuple or call this function
again
:param ham_dict: Hamiltonian to minimize
:param TN: TensorNetwork used as ansatz
:param tensor_order: the order of tensors. Can be a tuple with
integers or tuples of integers. Integer means 'optimize this
tensor', tuple means 'optimize these tensors jointly'
:param method: method of optimization. Can be "GP", "local"
:returns: value of the function. The `res` object is of limited
meaning, the parameters are stored in TN
"""
H = hamiltonians.explicit_hamiltonian(ham_dict)
if not isinstance(TN, TensorNetwork.TensorNetwork):
raise TypeError('Not a TensorNetwork')
q = QuantumRegister(TN.n_qubits)
c = ClassicalRegister(TN.n_qubits)
circ = QuantumCircuit(q, c)
block_size = TN.entangler.n_params
n_entanglers = TN.n_tensors
if tensor_order is None:
order = tuple(i for i in range(n_entanglers))
else:
order = tensor_order
for v in order:
if isinstance(v, int):
param_numbers = [v * block_size + i for i in range(block_size)]
elif isinstance(v, tuple):
param_numbers = []
for j in v:
param_numbers += [
j * block_size + i for i in range(block_size)
]
else:
raise TypeError('tensor_order should be a tuple or nested tuple')
print('Optimizing params', param_numbers, end=' -> ')
def f(x):
return arb_index_objective([float64(xi) for xi in x],
param_numbers,
ham_dict,
TN,
q,
c,
circuit=circ,
explH=H)
x0 = [TN.params[i] for i in param_numbers]
res = gbrt_minimize(f, [(0, 2 * pi)] * len(x0),
x0=x0,
n_calls=n_calls,
verbose=False)
#res = shgo(f, [(0, 4*pi)] * len(x0))
for i, num in enumerate(param_numbers):
TN.params[num] = res.x[i]
print('E = {0:0.4f}'.format(res.fun))
return res.fun
def main():
args = setup()
print("Preparing directories")
filename = f"{args.prefix}{args.model}_{args.data}_{args.estimator}{args.suffix}"
factors_path = os.path.join(args.root_dir, "factors", filename)
weights_path = os.path.join(args.root_dir, "weights",
f"{args.model}_{args.data}.pth")
if args.exp_id == -1:
if not args.no_results:
os.makedirs(os.path.join(args.results_dir, args.model, "data",
args.estimator, args.optimizer),
exist_ok=True)
if args.plot:
os.makedirs(os.path.join(args.results_dir, args.model, "figures",
args.estimator, args.optimizer),
exist_ok=True)
results_path = os.path.join(args.results_dir, args.model, "data",
args.estimator, args.optimizer, filename)
else:
if not args.no_results:
os.makedirs(os.path.join(args.results_dir, args.model, "data",
args.estimator, args.optimizer,
args.exp_id),
exist_ok=True)
if args.plot:
os.makedirs(os.path.join(args.results_dir, args.model, "figures",
args.estimator, args.optimizer,
args.exp_id),
exist_ok=True)
results_path = os.path.join(args.results_dir, args.model, "data",
args.estimator, args.optimizer,
args.exp_id, filename)
print("Loading model")
if args.model == 'lenet5':
model = lenet5(pretrained=args.data, device=args.device)
elif args.model == 'resnet18' and args.data != 'imagenet':
model = resnet18(pretrained=weights_path,
num_classes=43 if args.data == 'gtsrb' else 10,
device=args.device)
else:
model_class = getattr(torchvision.models, args.model)
if args.model in ['googlenet', 'inception_v3']:
model = model_class(pretrained=True, aux_logits=False)
else:
model = model_class(pretrained=True)
model.to(args.device).eval()
if args.parallel:
model = torch.nn.parallel.DataParallel(model)
print("Loading data")
if args.data == 'mnist':
val_loader = datasets.mnist(args.torch_data, splits='val')
elif args.data == 'cifar10':
val_loader = datasets.cifar10(args.torch_data, splits='val')
elif args.data == 'gtsrb':
val_loader = datasets.gtsrb(args.data_dir,
batch_size=args.batch_size,
splits='val')
elif args.data == 'imagenet':
img_size = 224
if args.model in ['googlenet', 'inception_v3']:
img_size = 299
val_loader = datasets.imagenet(args.data_dir,
img_size,
args.batch_size,
args.workers,
splits='val',
use_cache=True,
pre_cache=True)
print("Loading factors")
if args.estimator in ["diag", "kfac"]:
factors = torch.load(factors_path + '.pth')
elif args.estimator == 'efb':
kfac_factors = torch.load(factors_path.replace("efb", "kfac") + '.pth')
lambdas = torch.load(factors_path + '.pth')
factors = list()
eigvecs = get_eigenvectors(kfac_factors)
for eigvec, lambda_ in zip(eigvecs, lambdas):
factors.append((eigvec[0], eigvec[1], lambda_))
elif args.estimator == 'inf':
factors = torch.load(f"{factors_path}{args.rank}.pth")
torch.backends.cudnn.benchmark = True
norm_min = -10
norm_max = 10
scale_min = -10
scale_max = 10
if args.boundaries:
x0 = list()
boundaries = [[norm_min, scale_min], [norm_max, scale_max],
[norm_min, scale_max], [norm_max, scale_min],
[norm_min / 2., scale_min], [norm_max / 2., scale_max],
[norm_min, scale_max / 2.], [norm_max, scale_min / 2.],
[norm_min / 2., scale_min / 2.],
[norm_max / 2., scale_max / 2.],
[norm_min / 2., scale_max / 2.],
[norm_max / 2., scale_min / 2.]]
for b in boundaries:
tmp = list()
for _ in range(3 if args.layer else 1):
tmp.extend(b)
x0.append(tmp)
else:
x0 = None
f_norms = np.array([factor.norm().cpu().numpy() for factor in factors])
space = list()
for i in range(3 if args.layer else 1):
space.append(
skopt.space.Real(norm_min,
norm_max,
name=f"norm{i}",
prior='uniform'))
space.append(
skopt.space.Real(scale_min,
scale_max,
name=f"scale{i}",
prior='uniform'))
stats = {
"norms": [],
"scales": [],
"acc": [],
"ece": [],
"nll": [],
"ent": [],
"cost": []
}
@skopt.utils.use_named_args(dimensions=space)
def objective(**params):
norms = list()
scales = list()
for f in f_norms:
if args.layer:
# Closest to max
if abs(f_norms.max() - f) < abs(f_norms.min() - f) and abs(
f_norms.max() - f) < abs(f_norms.mean() - f):
norms.append(10**params['norm0'])
scales.append(10**params['scale0'])
# Closest to min
elif abs(f_norms.min() - f) < abs(f_norms.max() - f) and abs(
f_norms.min() - f) < abs(f_norms.mean() - f):
norms.append(10**params['norm1'])
scales.append(10**params['scale1'])
# Closest to mean
else:
norms.append(10**params['norm2'])
scales.append(10**params['scale2'])
else:
norms.append(10**params['norm0'])
scales.append(10**params['scale0'])
if args.layer:
print(
tabulate.tabulate(
{
'Layer': np.arange(len(factors)),
'F-Norm:': f_norms,
'Norms': norms,
'Scales': scales
},
headers='keys',
numalign='right'))
else:
print("Norm:", norms[0], "Scale:", scales[0])
try:
inv_factors = invert_factors(factors, norms,
args.pre_scale * scales,
args.estimator)
except (RuntimeError, np.linalg.LinAlgError):
print(f"Error: Singular matrix")
return 200
predictions, labels, _ = eval_bnn(model,
val_loader,
inv_factors,
args.estimator,
args.samples,
stats=False,
device=args.device,
verbose=False)
err = 100 - accuracy(predictions, labels)
ece = 100 * expected_calibration_error(predictions, labels)[0]
nll = negative_log_likelihood(predictions, labels)
ent = predictive_entropy(predictions, mean=True)
stats["norms"].append(norms)
stats["scales"].append(scales)
stats["acc"].append(100 - err)
stats["ece"].append(ece)
stats["nll"].append(nll)
stats["ent"].append(ent)
stats["cost"].append(err + ece)
print(
f"Err.: {err:.2f}% | ECE: {ece:.2f}% | NLL: {nll:.3f} | Ent.: {ent:.3f}"
)
return err + ece
with warnings.catch_warnings():
warnings.filterwarnings("ignore", category=FutureWarning)
if args.optimizer == "gbrt":
res = skopt.gbrt_minimize(func=objective,
dimensions=space,
n_calls=args.calls,
x0=x0,
verbose=True,
n_jobs=args.workers,
n_random_starts=0 if x0 else 10,
acq_func='EI')
# EI (neg. expected improvement)
# LCB (lower confidence bound)
# PI (neg. prob. of improvement): Usually favours exploitation over exploration
# gp_hedge (choose probabilistically between all)
if args.optimizer == "gp":
res = skopt.gp_minimize(func=objective,
dimensions=space,
n_calls=args.calls,
x0=x0,
verbose=True,
n_jobs=args.workers,
n_random_starts=0 if x0 else 1,
acq_func='gp_hedge')
# acq_func: EI (neg. expected improvement), LCB (lower confidence bound), PI (neg. prob. of improvement)
# xi: how much improvement one wants over the previous best values.
# kappa: Importance of variance of predicted values. High: exploration > exploitation
# base_estimator: RF (random forest), ET (extra trees)
elif args.optimizer == "forest":
res = skopt.forest_minimize(func=objective,
dimensions=space,
n_calls=args.calls,
x0=x0,
verbose=True,
n_jobs=args.workers,
n_random_starts=0 if x0 else 1,
acq_func='EI')
elif args.optimizer == "random":
res = skopt.dummy_minimize(func=objective,
dimensions=space,
n_calls=args.calls,
x0=x0,
verbose=True)
elif args.optimizer == "grid":
space = [
np.arange(norm_min, norm_max + 1, 10),
np.arange(scale_min, scale_max + 1, 10)
]
res = grid(func=objective, dimensions=space)
print(f"Minimal cost of {min(stats['cost'])} found at:")
if args.layer:
print(
tabulate.tabulate(
{
'Layer': np.arange(len(factors)),
'F-Norm:': f_norms,
'Norms': stats['norms'][np.argmin(stats['cost'])],
'Scales': stats['scales'][np.argmin(stats['cost'])]
},
headers='keys',
numalign='right'))
else:
print("Norm:", stats['norms'][np.argmin(stats['cost'])][0],
"Scale:", stats['scales'][np.argmin(stats['cost'])][0])
if not args.no_results:
print("Saving results")
del res.specs['args']['func']
np.save(
results_path +
f"_best_params{'_layer.npy' if args.layer else '.npy'}", [
stats['norms'][np.argmin(stats['cost'])],
stats['scales'][np.argmin(stats['cost'])]
])
np.save(
results_path +
f"_hyperopt_stats{'_layer.npy' if args.layer else '.npy'}", stats)
skopt.dump(
res, results_path +
f"_hyperopt_dump{'_layer.pkl' if args.layer else '.pkl'}")
if args.plot:
print("Plotting results")
hyperparameters(args)
def run(self,
exp_function: list or function,
params: dict = None,
artifacts: dict = None,
param_space=None,
max_evals=10) -> list:
"""
Runs the given experiment function or list of functions with various parameter configuration
# Arguments
exp_function (function or list): Method that implements the experiment.
params (dict): Dictionary that contains the configuration (e.g. hyperparameters) for the experiment (optional).
artifacts (dict): Dictionary that contains artifacts (any kind of python object) required for the experiment (optional).
param_space (iterator): Parameter Space suited for Skopt Library.
max_evals (int): Maximum number of evaluations. Default: 10.
# Returns
List of finished experiments
"""
from skopt.utils import use_named_args
@use_named_args(param_space)
def skopt_objective(**param_selection):
exp = self.create_exp(
name_suffix=text_utils.simplify_dict_to_str(param_selection))
params.update(param_selection)
exp.run_exp(exp_function, params, artifacts)
score = exp.exp_metadata.result
if not score or not isinstance(score, numbers.Number):
exp.log.info(
"Provided exp_function did not return a numeric score/result."
)
return None
if self.maximize_score:
# hyperopt only supports min
score = -score
return score
if self.skopt_algo == "gp":
from skopt import gp_minimize
self.skopt_result = gp_minimize(skopt_objective,
param_space,
n_calls=max_evals,
**self.skopt_args)
elif self.skopt_algo == "gbrt":
from skopt import gbrt_minimize
self.skopt_result = gbrt_minimize(skopt_objective,
param_space,
n_calls=max_evals,
**self.skopt_args)
elif self.skopt_algo == "forest":
from skopt import forest_minimize
self.skopt_result = forest_minimize(skopt_objective,
param_space,
n_calls=max_evals,
**self.skopt_args)
elif self.skopt_algo == "dummy":
from skopt import dummy_minimize
self.skopt_result = dummy_minimize(skopt_objective,
param_space,
n_calls=max_evals,
**self.skopt_args)
else:
self.log.warning("The selected algorithm is unknown: " +
self.skopt_algo)
return self.experiments()
def local_lipschitz_estimate(self,
x,
optim='gp',
eps=None,
bound_type='box',
clip=True,
n_calls=100,
njobs=-1,
verbose=False):
"""
Compute one-sided lipschitz estimate for explainer. Adequate for local
Lipschitz, for global must have the two sided version. This computes:
max_z || f(x) - f(z)|| / || x - z||
Instead of:
max_z1,z2 || f(z1) - f(z2)|| / || z1 - z2||
If eps provided, does local lipzshitz in:
- box of width 2*eps along each dimension if bound_type = 'box'
- box of width 2*eps*va, along each dimension if bound_type = 'box_norm' (i.e. normalize so that deviation is eps % in each dim )
- box of width 2*eps*std along each dimension if bound_type = 'box_std'
max_z || f(x) - f(z)|| / || x - z|| , with f = theta
clip: clip bounds to within (min, max) of dataset
"""
# Compute bounds for optimization
if eps is None:
# If want to find global lipzhitz ratio maximizer - search over "all space" - use max min bounds of dataset fold of interest
# gp can't have lower bound equal upper bound - so move them slightly appart
lwr = self.train_stats['min'].flatten() - 1e-6
upr = self.train_stats['max'].flatten() + 1e-6
elif bound_type == 'box':
lwr = (x - eps).flatten()
upr = (x + eps).flatten()
elif bound_type == 'box_std':
# gp can't have lower bound equal upper bound - so set min std to 0.001
lwr = (x -
eps * np.maximum(self.train_stats['std'], 0.001)).flatten()
upr = (x +
eps * np.maximum(self.train_stats['std'], 0.001)).flatten()
if clip:
lwr = lwr.clip(min=self.train_stats['min'].min())
upr = upr.clip(max=self.train_stats['max'].max())
bounds = list(zip(*[lwr, upr]))
if x.ndim > 2:
# This is an image, will need to reshape
orig_shape = x.shape
x = x.flatten()
else:
orig_shape = x.shape
# Run optimization
if optim == 'gp':
print('Running BlackBox Minimization with Bayesian Optimization')
# Need minus because gp only has minimize method
f = partial(self.lipschitz_ratio,
x,
reshape=orig_shape,
minus=True)
res = gp_minimize(f,
bounds,
n_calls=n_calls,
verbose=verbose,
n_jobs=njobs)
elif optim == 'gbrt':
print('Running BlackBox Minimization with Gradient Boosted Trees')
f = partial(self.lipschitz_ratio,
x,
reshape=orig_shape,
minus=True)
res = gbrt_minimize(f,
bounds,
n_calls=n_calls,
verbose=verbose,
n_jobs=njobs)
lip, x_opt = -res['fun'], np.array(res['x'])
if verbose:
print(lip, np.linalg.norm(x - x_opt))
return lip, x_opt.reshape(orig_shape)
def main():
# Evaluate command line arguments
cd = os.path.dirname(os.path.abspath(__file__))
parser = argparse.ArgumentParser(
description="Performs point cloud registration.")
parser.add_argument("-c",
"--config",
default=os.path.join(cd, "hyperopt.ini"),
type=str,
help="/path/to/hyperopt.ini.")
parser.add_argument("-v",
"--verbose",
action="store_true",
help="Get verbose output during execution.")
parser.add_argument("-d",
"--draw",
action="store_true",
help="Visualize results results.")
parser.add_argument("-o",
"--output",
default=os.path.join(cd, "output"),
type=str,
help="/path/to/output/dir")
args = parser.parse_args()
# Make output directory
os.makedirs(args.output, exist_ok=True)
# Read hyperparameter optimization config (hyper config) from argument
hyper_config = configparser.ConfigParser(inline_comment_prefixes='#')
hyper_config.read(args.config)
# Set frequent config options
optimization = hyper_config["optimization"]
optimizer = optimization.get("optimizer").lower()
options = hyper_config["options"]
# Read run registration config from file
run_config = configparser.ConfigParser(inline_comment_prefixes='#')
run_config.read(os.path.join(cd, "registration.ini"))
# Enable verbose output
verbose = args.verbose or options.getboolean("verbose")
if verbose:
logger.setLevel(logging.DEBUG)
print_config(hyper_config)
# Set run config with values from hyper config
set_config_params_with_config_or_dict(config_or_dict_from=hyper_config,
config_to=run_config)
run_config["options"]["verbose"] = str(False)
run_config["options"]["progress"] = str(False)
run_config["options"]["print_results"] = str(False)
run_config["options"][
"return"] = "results+errors" if "error" in optimization.get(
"minimize").lower() else "results"
run_config["options"]["use_degrees"] = str(False)
configs = list()
# Create a Scikit-Optimize search space from the hyper config
space = get_skopt_space_from_config(config=hyper_config)
# Make a progress bar
progress = tqdm.tqdm(range(optimization.getint("n_calls")),
desc=f"Optimization ({optimizer.upper()})",
file=sys.stdout,
disable=not options.getboolean("progress") or verbose)
def progress_callback(_result: skopt.utils.OptimizeResult) -> None:
progress.set_postfix_str(
f"Current minimum ({optimization.get('minimize')}): {np.min(_result.func_vals)}"
)
progress.update()
# Make a checkpointer
def checkpoint_callback(_result: skopt.utils.OptimizeResult) -> None:
try:
if len(_result.x_iters) % 10 == 1:
_res = copy.deepcopy(_result)
del _res.specs['args']['callback']
del _res.specs['args']['func']
skopt.dump(res=_res,
filename=os.path.join(args.output,
"checkpoint.pkl"),
compress=True)
except Exception as ex:
message = f"Couldn't save checkpoint due to exception: {ex}. Skipping."
logger.exception(message) if progress.disable else progress.write(
message)
# The optimization objective function
@skopt.utils.use_named_args(dimensions=space)
def objective(**params: Dict[str,
Any]) -> Union[float, Tuple[float, float]]:
start = time.time()
if verbose:
print(
tabulate.tabulate(
{
"Option": list(params.keys()),
"Values": list(params.values())
},
headers="keys",
missingval="None"))
set_config_params_with_config_or_dict(
config_or_dict_from=params,
config_to=run_config,
sections_to_skip=["DEFAULT", "optimization", "options", "data"])
configs.append(copy.deepcopy(run_config))
try:
results = run(config=run_config)
if any("error" in key for key in results.keys()):
if optimization.get("minimize").lower() in [
"errors", "errors_rot+errors_trans"
]:
cost = sum(results["errors_rot"] + results["errors_trans"])
elif optimization.get("minimize").lower() == "errors_rot":
cost = sum(results["errors_rot"])
elif optimization.get("minimize").lower() == "errors_trans":
cost = sum(results["errors_trans"])
else:
raise ValueError(
f"Invalid config option '{optimization.get('minimize')}' for 'minimize'."
)
else:
if optimization.get("minimize").lower() in [
"errors", "errors_rot+errors_trans"
]:
message = f"Optimization objective is '{optimization.get('minimize')}' which was not found in" \
f"returned results. Did you provide ground truth data? Using inlier RMSE instead."
raise ValueError(message)
elif optimization.get(
"minimize").lower() == "inlier_rmse-fitness":
cost = sum([
r.inlier_rmse - r.fitness for r in results["results"]
])
elif optimization.get("minimize").lower() in [
"fitness", "-fitness"
]:
cost = -sum([r.fitness for r in results["results"]])
elif optimization.get("minimize").lower() == "inlier_rmse":
cost = sum([r.inlier_rmse for r in results["results"]])
else:
raise ValueError(
f"Invalid config option '{optimization.get('minimize')}' for 'minimize'."
)
if optimization.getboolean("scale_objective"):
num_correspondences = sum([
len(r.correspondence_set) for r in results["results"]
])
cost = cost / num_correspondences if num_correspondences > 0 else 1e30
except Exception as ex:
message = f"Caught exception during execution of 'run_registration.run': {ex}"
logger.exception(message) if progress.disable else progress.write(
message)
cost = 1e30
if not np.isfinite(cost):
cost = 1e30
if optimizer not in [
"rnd", "random"
] and optimization.get("acq_func").lower() in ["eips", "pips"]:
return cost, time.time() - start
else:
return cost
# Load initial points
init = optimization.get("init_from_file_or_list")
x0, y0 = None, None
try:
init = eval(init)
if init is not None and isinstance(init, (list, tuple)):
x0 = init
y0 = None
except (NameError, SyntaxError):
try:
init = skopt.load(init)
x0 = init.x_iters
y0 = init.func_vals
except FileNotFoundError:
logger.exception(
f"Couldn't load initial points from {init}. Skipping.")
x0 = None
y0 = None
# Run optimization
if optimizer in ["rnd", "rand", "random"]:
result = skopt.dummy_minimize(
func=objective,
dimensions=space,
n_calls=optimization.getint("n_calls"),
initial_point_generator=optimization.get(
"initial_point_generator"),
x0=x0,
y0=y0,
random_state=eval(optimization.get("random_state")),
verbose=verbose,
callback=[progress_callback, checkpoint_callback])
elif all(c in optimizer for c in list("rf")):
result = skopt.forest_minimize(
func=objective,
dimensions=space,
base_estimator=optimization.get("base_estimator").upper(),
n_calls=optimization.getint("n_calls"),
n_initial_points=optimization.getint("n_initial_points"),
acq_func=optimization.get("acq_func"),
initial_point_generator=optimization.get(
"initial_point_generator"),
x0=x0,
y0=y0,
random_state=eval(optimization.get("random_state")),
verbose=verbose,
callback=[progress_callback, checkpoint_callback],
n_points=optimization.getint("n_points"),
xi=optimization.getfloat("xi"),
kappa=optimization.getfloat("kappa"),
n_jobs=optimization.getint("n_jobs"))
elif all(c in optimizer for c in list("gp")):
noise = optimization.get("noise").lower()
noise = noise if "gauss" in noise else optimization.getfloat("noise")
result = skopt.gp_minimize(
func=objective,
dimensions=space,
n_calls=optimization.getint("n_calls"),
n_initial_points=optimization.getint("n_initial_points"),
acq_func=optimization.get("acq_func"),
acq_optimizer=optimization.get("acq_optimizer"),
initial_point_generator=optimization.get(
"initial_point_generator"),
x0=x0,
y0=y0,
random_state=eval(optimization.get("random_state")),
verbose=verbose,
callback=[progress_callback, checkpoint_callback],
n_points=optimization.getint("n_points"),
n_restarts_optimizer=optimization.getint("n_restarts_optimizer"),
xi=optimization.getfloat("xi"),
kappa=optimization.getfloat("kappa"),
noise=noise,
n_jobs=optimization.getint("n_jobs"))
elif all(c in optimizer for c in list("gbrt")):
result = skopt.gbrt_minimize(
func=objective,
dimensions=space,
n_calls=optimization.getint("n_calls"),
n_initial_points=optimization.getint("n_initial_points"),
acq_func=optimization.get("acq_func"),
initial_point_generator=optimization.get(
"initial_point_generator"),
x0=x0,
y0=y0,
random_state=eval(optimization.get("random_state")),
verbose=verbose,
callback=[progress_callback, checkpoint_callback],
n_points=optimization.getint("n_points"),
xi=optimization.getfloat("xi"),
kappa=optimization.getfloat("kappa"),
n_jobs=optimization.getint("n_jobs"))
else:
raise ValueError(f"Invalid optimizer {optimizer}.")
progress.close()
# Print results
names = None
dims = None
try:
res = np.concatenate(
[np.expand_dims(result.func_vals, axis=1), result.x_iters], axis=1)
iters = np.argsort(res[:, 0])
table_values = res[iters]
names = [dim.name for dim in space]
dims = [f"X_{i}"
for i in range(len(names))] if len(names) > 10 else names
table_headers = ["Iter", "Cost"] + dims
print()
print("OPTIMIZATION RESULTS:\n====================")
print(
tabulate.tabulate(table_values,
headers=table_headers,
missingval="None",
showindex=list(iters)))
if len(names) > 10:
print()
print("DIMENSIONS <-> NAMES:\n====================")
print(
tabulate.tabulate({
"Dimension": dims,
"Name": names
},
headers="keys"))
except Exception as e:
logger.exception(
f"Printing of results failed due to exception: {e}. Trying to save results."
)
# Visualize results
if (args.draw or options.getboolean("draw")) and names is not None:
try:
if len(names) > 5:
logger.info(
f"Drawing visualizations for upt to 10 dimensions. This can take some time."
)
from matplotlib import pyplot as plt
plot_convergence(result)
try:
plot_dims = eval(options.get("plot_dims"))
if isinstance(plot_dims, float):
plot_dims = int(plot_dims)
elif isinstance(plot_dims, int) or plot_dims is None:
pass
else:
raise TypeError(
f"'plot_dims' must by None or int but is {type(plot_dims)}. Skipping selection."
)
except (NameError, SyntaxError, TypeError):
plot_dims = None
if (plot_dims is None
and len(names) > 10) or (isinstance(plot_dims, int)
and plot_dims > 10):
logger.info(
f"Too many dimensions ({len(names)}) for 'evaluations' and 'objective' plots. "
f"Only drawing the first 10.")
plot_dims = 10
if isinstance(plot_dims, int):
plot_dims = names[:plot_dims]
else:
plot_dims = None
plot_evaluations(
result,
dimensions=dims if plot_dims is None else plot_dims,
plot_dims=plot_dims)
if optimizer not in ["rnd", "rand", "random"]:
plot_objective(
result,
dimensions=dims if plot_dims is None else plot_dims,
plot_dims=plot_dims)
else:
logger.info(
f"Can't plot 'objective' for optimizer '{optimizer}'.")
plt.show()
except Exception as e:
logger.exception(
f"Drawing failed due to exception: {e}. Skipping.")
# Save best result config
filename = None
config_hash = None
if options.getboolean("save") or names is None:
try:
filename = options.get("filename")
try:
filename = eval(filename)
except (NameError, SyntaxError):
if filename.lower() == "none":
filename = None
best_config = configs[int(np.argmin(result.func_vals))]
best_config["options"]["progress"] = str(True)
best_config["options"]["print_results"] = str(True)
if filename is None:
if options.getboolean("overwrite"):
config_hash = str().join([
value for option, value in hyper_config.items("data")
])
else:
config_hash = str()
for section in hyper_config.sections():
for option, value in hyper_config.items(section):
config_hash += value
config_hash = hashlib.md5(
config_hash.encode('utf-8')).hexdigest()
output_path = os.path.join(args.output, f"{config_hash}.ini")
else:
output_path = os.path.join(args.output,
f"{filename.split('.')[0]}.ini")
with open(output_path, 'w') as configfile:
best_config.write(configfile)
except Exception as e:
config_hash = hex(random.getrandbits(128))
logger.exception(
f"Saving of best result failed due to exception: {e}. Trying to dump all results."
)
# Save results
if filename is None:
output_path = os.path.join(args.output, f"{config_hash}.pkl")
else:
output_path = os.path.join(args.output,
f"{filename.split('.')[0]}.pkl")
try:
del result.specs['args']['callback']
skopt.dump(res=result, filename=output_path, compress=True)
except Exception as e:
logger.debug(f"Caught exception during 'skopt.dump': {e}")
logger.debug("Trying to store the result without the objective.")
skopt.dump(res=result,
filename=output_path,
store_objective=False,
compress=True)
finally:
logger.debug("Deleting the objective.")
del result.specs['args']['func']
skopt.dump(res=result, filename=output_path, compress=True)
def hyperdrive(objective, hyperparameters, results_path, model="GP", n_iterations=50, verbose=False,
checkpoints_path=None, deadline=None, sampler=None, n_samples=None, random_state=0):
"""
Distributed optimization - one optimization per node.
Parameters
----------
* `objective` [function]:
User defined function which calls a learner
and returns a metric of interest.
* `hyperparameters` [list, shape=(n_hyperparameters,)]:
* `results_path` [string]
Path to save optimization results
* `checkpoint_path` [string]
Path to previously saved results. Used to resume optimization.
* `model` [string, default="GP"]
Probilistic learner used to model our objective function.
Options:
- "GP": Gaussian process
- "RF": Random forest
- "GBRT": Gradient boosted regression trees
- "RAND": Random search
* `n_iterations` [int, default=50]
Number of optimization iterations
* `verbose` [bool, default=False]
Verbosity of optimization.
* `checkpoints` [bool, default=False]
Whether to checkpoint at each step of the optimization.
* `deadline` [int, optional]
Deadline (seconds) for the optimization to finish within.
* `sampler` [str, default=None]
Random sampling scheme for optimizer's initial runs.
Options:
- "lhs": latin hypercube sampling
* `n_samples` [int, default=None]
Number of random samples to be drawn from the `sampler`.
- Required if you would like to use `sampler`.
- Must be <= the number of elements in the smallest hyperparameter bound's set.
* `random_state` [int, default=0]
Random state for reproducibility.
"""
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
size = comm.Get_size()
if checkpoints_path and sampler:
raise ValueError('Cannot use both a restart from a previous run and ' \
'use latin hypercube sampling for initial search points!')
# Setup savefile
if rank < 10:
# Ensure results are sorted by rank
filename = 'hyperspace' + str(0) + str(rank)
else:
filename = 'hyperspace' + str(rank)
savefile = os.path.join(results_path, filename)
# Create hyperspaces, and either sampling bounds or checkpoints
hyperspace = create_hyperspace(hyperparameters)
space = hyperspace[rank]
# Latin hypercube sampling
if sampler and not n_samples:
raise ValueError(f'Sampler requires n_samples > 0. Got {n_samples}')
elif sampler and n_samples:
hyperbounds = create_hyperbounds(hyperparameters)
bounds = hyperbounds[rank]
# Get initial points in domain via latin hypercube sampling
init_points = lhs_start(bounds, n_samples)
init_response = None
n_rand = 10 - len(init_points)
else:
init_points = None
init_response = None
n_rand = 10
# Resuming from checkpoint
if checkpoints_path:
checkpoint = _load_checkpoint(checkpoints_path, rank)
try:
init_points = checkpoint.x_iters
init_response = checkpoint.func_vals
n_rand = 10 - len(init_points)
except AttributeError:
# Missing saves won't have initial values.
init_points = None
init_response = None
n_rand = 10
callbacks = []
if deadline:
deadline = DeadlineStopper(deadline)
callbacks.append(deadline)
if checkpoints_path:
checkpoint_callback = CheckpointSaver(checkpoints_path, filename)
callbacks.append(checkpoint_callback)
# Thanks Guido for refusing to believe in switch statements.
# Case 0
if model == "GP":
# Verbose mode should only run on node 0.
if verbose and rank == 0:
result = gp_minimize(objective, space, n_calls=n_iterations, verbose=verbose,
callback=callbacks, x0=init_points, y0=init_response,
n_random_starts=n_rand, random_state=random_state)
else:
result = gp_minimize(objective, space, n_calls=n_iterations,
callback=callbacks, x0=init_points, y0=init_response,
n_random_starts=n_rand, random_state=random_state)
# Case 1
elif model == "RF":
if verbose and rank == 0:
result = forest_minimize(objective, space, n_calls=n_iterations, verbose=verbose,
callback=callbacks, x0=init_points, y0=init_response,
n_random_starts=n_rand, random_state=random_state)
else:
result = forest_minimize(objective, space, n_calls=n_iterations,
callback=callbacks, x0=init_points, y0=init_response,
n_random_starts=n_rand, random_state=random_state)
# Case 2
elif model == "GBRT":
if verbose and rank == 0:
result = gbrt_minimize(objective, space, n_calls=n_iterations, verbose=verbose,
callback=callbacks, x0=init_points, y0=init_response,
n_random_starts=n_rand, random_state=random_state)
else:
result = gbrt_minimize(objective, space, n_calls=n_iterations,
callback=callbacks, x0=init_points, y0=init_response,
n_random_starts=n_rand, random_state=random_state)
# Case 3
elif model == "RAND":
if verbose and rank == 0:
result = dummy_minimize(objective, space, n_calls=n_iterations, verbose=verbose,
callback=callbacks, x0=init_points, y0=init_response,
random_state=random_state)
else:
result = dummy_minimize(objective, space, n_calls=n_iterations,
callback=callbacks, x0=init_points, y0=init_response,
random_state=random_state)
else:
raise ValueError("Invalid model {}. Read the documentation for "
"supported models.".format(model))
# Each worker will independently write their results to disk
dump(result, savefile)
print('\n')
lgb_mod = lgb.LGBMClassifier(**params)
best_mod = lgb_mod.fit(X_train,y_train,**fit_params)
y_pred = best_mod.predict(X_test)
print('y_pred.value_counts() -> \n{}'.format(np.unique(y_pred,return_counts=True)))
loss = lgbm_custom_loss(y_test,y_pred)
if loss[2] == True:#higher is better
loss = 1 - loss[1]
else:
loss = loss[1]
print('loss -> {}'.format(loss))
return loss
HPO_PARAMS = {
'n_calls':100
}
results = skopt.gbrt_minimize(train_func,CONFIG['lgbm_training']['skopt_params'],**HPO_PARAMS)
print('\nBEST RESULT - {} -> {:.4f}'.format(CONFIG['lgbm_training']['custom_metric'],results.fun))
best_params = {k.name:v for k, v in zip(CONFIG['lgbm_training']['skopt_params'],results.x)}
print('\nBEST PARAMS -> {}'.format(best_params))
run_time.end()
### FIT THE FINAL MODEL USIN BEST PARAMS ###
final_models = lgb.LGBMClassifier(**best_params)
final_models.fit(X_train,y_train)
#Show the best and worst features
lgb.plot_importance(final_models,figsize=(12,36))
#Run on the testing set
y_pred = final_models.predict(X_test)
print(calc_tpr(y_pred,y_test,['FINAL RF MODEL - TEST SET'],y_train.unique()))
import matplotlib.pyplot as plt
from skopt import gbrt_minimize
from skopt.benchmarks import bench3
from skopt.acquisition import gaussian_ei
dimensions = [(-1.0, 1.0)]
x = np.linspace(-1, 1, 200)
func_values = [bench3(xi) for xi in x]
plt.figure(figsize=(10, 5))
vals = np.reshape(x, (-1, 1))
col_no = 1
row_no = 6
res = gbrt_minimize(
bench3, dimensions, maxiter=6, n_start=1, random_state=1)
best_xs = res.x_iters.ravel()
best_ys = res.func_vals.ravel()
models = res.models
for n_iter in range(5):
model = models[n_iter]
best_x = best_xs[:n_iter+1]
best_y = best_ys[:n_iter+1]
low, mu, high = model.predict(vals).T
std = (high - low) / 2
acquis_values = -gaussian_ei(vals, model, best_y[-1])
acquis_values = acquis_values.ravel()
posterior_mean = mu.ravel()
posterior_std = std.ravel()
def hyperdrive(objective,
hyperparameters,
results_path,
model="GP",
n_iterations=50,
verbose=False,
deadline=None,
sampler=None,
n_samples=None,
random_state=0):
"""
Distributed optimization - one optimization per node.
Parameters
----------
* `objective` [function]:
User defined function which calls a learner
and returns a metric of interest.
* `hyperparameters` [list, shape=(n_hyperparameters,)]:
* `results_path` [string]
Path to save optimization results
* `model` [string, default="GP"]
Probilistic learner used to model our objective function.
Options:
- "GP": Gaussian process
- "RF": Random forest
- "GBRT": Gradient boosted regression trees
- "RAND": Random search
* `n_iterations` [int, default=50]
Number of optimization iterations
* `verbose` [bool, default=False]
Verbosity of optimization.
* `deadline` [int, optional]
Deadline (seconds) for the optimization to finish within.
* `sampler` [str, default=None]
Random sampling scheme for optimizer's initial runs.
Options:
- "lhs": latin hypercube sampling
* `n_samples` [int, default=None]
Number of random samples to be drawn from the `sampler`.
- Required if you would like to use `sampler`.
- Must be <= the number of elements in the smallest hyperparameter bound's set.
* `random_state` [int, default=0]
Random state for reproducibility.
"""
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
size = comm.Get_size()
if rank == 0:
hyperspace = create_hyperspace(hyperparameters)
if sampler and not n_samples:
raise ValueError(
'Sampler requires n_samples > 0. Got {}'.format(n_samples))
elif sampler and n_samples:
hyperbounds = create_hyperbounds(hyperparameters)
else:
hyperspace = None
if sampler is not None:
hyperbounds = None
space = comm.scatter(hyperspace, root=0)
if sampler:
bounds = comm.scatter(hyperbounds, root=0)
# Get initial points in the obj. function domain via latin hypercube sampling
init_points = lhs_start(bounds, n_samples)
n_rand = 10 - len(init_points)
else:
init_points = None
n_rand = 10
if deadline:
deadline = DeadlineStopper(deadline)
# Thanks Guido for refusing to believe in switch statements.
# Case 0
if model == "GP":
# Verbose mode should only run on node 0.
if verbose and rank == 0:
result = gp_minimize(objective,
space,
n_calls=n_iterations,
verbose=verbose,
callback=deadline,
x0=init_points,
n_random_starts=n_rand,
random_state=random_state)
else:
result = gp_minimize(objective,
space,
n_calls=n_iterations,
callback=deadline,
x0=init_points,
n_random_starts=n_rand,
random_state=random_state)
# Case 1
elif model == "RF":
if verbose and rank == 0:
result = forest_minimize(objective,
space,
n_calls=n_iterations,
verbose=verbose,
callback=deadline,
x0=init_points,
n_random_starts=n_rand,
random_state=random_state)
else:
result = forest_minimize(objective,
space,
n_calls=n_iterations,
callback=deadline,
x0=init_points,
n_random_starts=n_rand,
random_state=random_state)
# Case 2
elif model == "GRBRT":
if verbose and rank == 0:
result = gbrt_minimize(objective,
space,
n_calls=n_iterations,
verbose=verbose,
callback=deadline,
x0=init_points,
n_random_starts=n_rand,
random_state=random_state)
else:
result = gbrt_minimize(objective,
space,
n_calls=n_iterations,
callback=deadline,
x0=init_points,
n_random_starts=n_rand,
random_state=random_state)
# Case 3
elif model == "RAND":
if verbose and rank == 0:
result = dummy_minimize(objective,
space,
n_calls=n_iterations,
verbose=verbose,
callback=deadline,
x0=init_points,
n_random_starts=n_rand,
random_state=random_state)
else:
result = dummy_minimize(objective,
space,
n_calls=n_iterations,
callback=deadline,
x0=init_points,
n_random_starts=n_rand,
random_state=random_state)
else:
raise ValueError("Invalid model {}. Read the documentation for "
"supported models.".format(model))
# Each worker will independently write their results to disk
dump(result, results_path + '/hyperspace' + str(rank))
