def setUpClass(cls):
super(TestGPRGP, cls).setUpClass()
boston = datasets.load_boston()
data = boston['data']
X_train = data[0:500]
X_test = data[500:501]
y_train = boston['target'][0:500].reshape(500, 1)
X_min = np.min(X_train, 0)
X_max = np.max(X_train, 0)
random.seed(0)
np.random.seed(0)
tf.set_random_seed(0)
model_kwargs = {}
opt_kwargs = {}
opt_kwargs['learning_rate'] = 0.01
opt_kwargs['maxiter'] = 10
opt_kwargs['bounds'] = [X_min, X_max]
opt_kwargs['ucb_beta'] = 1.0
tf.reset_default_graph()
graph = tf.get_default_graph()
gpflow.reset_default_session(graph=graph)
cls.m = gpr_models.create_model('BasicGP',
X=X_train,
y=y_train,
**model_kwargs)
cls.gpr_result = tf_optimize(cls.m.model, X_test, **opt_kwargs)
def use_gpu(id):
os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
# The GPU id to use, usually either "0" or "1", "2' "3" "4"
os.environ["CUDA_VISIBLE_DEVICES"] = str(id)
# without using any gpu
# os.environ["CUDA_VISIBLE_DEVICES"]=""
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
# less memeory
gpflow.reset_default_session(config=config)
def setUpClass(cls):
super(TestGPRGPFlow, cls).setUpClass()
boston = datasets.load_boston()
data = boston['data']
X_train = data[0:500]
X_test = data[500:]
y_train = boston['target'][0:500].reshape(500, 1)
model_kwargs = {'lengthscales': 1, 'variance': 1, 'noise_variance': 1}
tf.reset_default_graph()
graph = tf.get_default_graph()
gpflow.reset_default_session(graph=graph)
cls.m = gpr_models.create_model('BasicGP', X=X_train, y=y_train,
**model_kwargs)
cls.gpr_result = gpflow_predict(cls.m.model, X_test)
def run_optimize(X, y, X_samples, model_name, opt_kwargs, model_kwargs):
timer = TimerStruct()
# Create model (this also optimizes the hyperparameters if that option is enabled
timer.start()
tf.reset_default_graph()
graph = tf.get_default_graph()
gpflow.reset_default_session(graph=graph)
m = gpr_models.create_model(model_name, X=X, y=y, **model_kwargs)
timer.stop()
model_creation_sec = timer.elapsed_seconds
LOG.info(m.model.as_pandas_table())
# Optimize the DBMS's configuration knobs
timer.start()
res = tf_optimize(m.model, X_samples, **opt_kwargs)
timer.stop()
config_optimize_sec = timer.elapsed_seconds
return res.minl_conf, res.minl, m.get_model_parameters(), m.get_hyperparameters()
def test_different_sessions(self):
with self.test_context():
demo = Demo()
# Force initialization.
with self.test_context() as session1:
gpflow.reset_default_session()
opt = self.optimizer()
opt.minimize(demo, maxiter=1)
# Mild initialization requirement: default changed session case.
with self.test_context() as session2:
self.assertFalse(session1 == session2)
gpflow.reset_default_session()
opt = self.optimizer()
opt.minimize(demo, maxiter=1, initialize=False)
# Mild initialization requirement: pass session case.
with self.test_context() as session3:
opt = self.optimizer()
opt.minimize(demo, maxiter=1, session=session3, initialize=False)
def test_different_sessions(self):
with self.test_context() as session:
demo = Demo()
# Force initialization.
with self.test_context() as session1:
gpflow.reset_default_session()
opt = self.optimizer()
demo.initialize(session1, force=True)
opt.minimize(demo, maxiter=1)
# Mild initialization requirement: default changed session case.
with self.test_context() as session2:
self.assertFalse(session1 == session2)
gpflow.reset_default_session()
opt = self.optimizer()
opt.minimize(demo, maxiter=1, initialize=True)
# Mild initialization requirement: pass session case.
with self.test_context() as session3:
opt = self.optimizer()
opt.minimize(demo, maxiter=1, session=session3, initialize=True)
def test_external_variables_in_model(self):
with self.test_context():
dfloat = gpflow.settings.float_type
var1_init = np.array(1.0, dtype=dfloat)
var2_init = np.array(2.0, dtype=dfloat)
var3_init = np.array(3.0, dtype=dfloat)
var1 = tf.get_variable('var1', initializer=var1_init, dtype=dfloat)
var2 = tf.get_variable('var2', initializer=var2_init, dtype=dfloat)
var3 = tf.get_variable('var3',
initializer=var3_init,
trainable=False,
dtype=dfloat)
opt = self.optimizer()
# Just initialize variable, but do not use it in training
with self.test_context() as session:
gpflow.reset_default_session()
demo1 = Demo(add_to_inits=[var1],
add_to_trainables=[],
name="demo1")
opt.minimize(demo1, maxiter=10)
self.assertEqual(session.run(var1), var1_init)
with self.test_context() as session:
gpflow.reset_default_session()
with self.assertRaises(tf.errors.FailedPreconditionError):
demo2 = Demo(add_to_inits=[],
add_to_trainables=[var2],
name="demo2")
opt.minimize(demo2, maxiter=10)
with self.test_context() as session:
gpflow.reset_default_session()
demo3 = Demo(add_to_inits=[var1, var2, var3],
add_to_trainables=[var2, var3],
name="demo3")
opt.minimize(demo3, maxiter=10)
self.assertEqual(session.run(var1), var1_init)
self.assertFalse(session.run(var3) == var3_init)
self.assertFalse(session.run(var2) == var2_init)
def test_external_variables_in_model(self):
with self.test_context():
dfloat = gpflow.settings.float_type
var1_init = np.array(1.0, dtype=dfloat)
var2_init = np.array(2.0, dtype=dfloat)
var3_init = np.array(3.0, dtype=dfloat)
var1 = tf.get_variable('var1', initializer=var1_init, dtype=dfloat)
var2 = tf.get_variable('var2', initializer=var2_init, dtype=dfloat)
var3 = tf.get_variable('var3', initializer=var3_init, trainable=False, dtype=dfloat)
opt = self.optimizer()
# Just initialize variable, but do not use it in training
with self.test_context() as session:
gpflow.reset_default_session()
demo1 = Demo(add_to_inits=[var1], add_to_trainables=[], name="demo1")
opt.minimize(demo1, maxiter=10)
self.assertEqual(session.run(var1), var1_init)
with self.test_context() as session:
gpflow.reset_default_session()
with self.assertRaises(tf.errors.FailedPreconditionError):
demo2 = Demo(add_to_inits=[], add_to_trainables=[var2], name="demo2")
opt.minimize(demo2, maxiter=10)
with self.test_context() as session:
gpflow.reset_default_session()
demo3 = Demo(add_to_inits=[var1, var2, var3],
add_to_trainables=[var2, var3], name="demo3")
opt.minimize(demo3, maxiter=10)
self.assertEqual(session.run(var1), var1_init)
self.assertFalse(session.run(var3) == var3_init)
self.assertFalse(session.run(var2) == var2_init)
def configuration_recommendation(recommendation_input):
target_data, algorithm = recommendation_input
LOG.info('configuration_recommendation called')
if target_data['bad'] is True:
target_data_res = dict(
status='bad',
result_id=target_data['newest_result_id'],
info='WARNING: no training data, the config is generated randomly',
recommendation=target_data['config_recommend'],
pipeline_run=target_data['pipeline_run'])
LOG.debug('%s: Skipping configuration recommendation.\n\ndata=%s\n',
AlgorithmType.name(algorithm),
JSONUtil.dumps(target_data, pprint=True))
return target_data_res
# Load mapped workload data
mapped_workload_id = target_data['mapped_workload'][0]
latest_pipeline_run = PipelineRun.objects.get(
pk=target_data['pipeline_run'])
mapped_workload = Workload.objects.get(pk=mapped_workload_id)
workload_knob_data = PipelineData.objects.get(
pipeline_run=latest_pipeline_run,
workload=mapped_workload,
task_type=PipelineTaskType.KNOB_DATA)
workload_knob_data = JSONUtil.loads(workload_knob_data.data)
workload_metric_data = PipelineData.objects.get(
pipeline_run=latest_pipeline_run,
workload=mapped_workload,
task_type=PipelineTaskType.METRIC_DATA)
workload_metric_data = JSONUtil.loads(workload_metric_data.data)
newest_result = Result.objects.get(pk=target_data['newest_result_id'])
cleaned_workload_knob_data = clean_knob_data(
workload_knob_data["data"], workload_knob_data["columnlabels"],
newest_result.session)
X_workload = np.array(cleaned_workload_knob_data[0])
X_columnlabels = np.array(cleaned_workload_knob_data[1])
y_workload = np.array(workload_metric_data['data'])
y_columnlabels = np.array(workload_metric_data['columnlabels'])
rowlabels_workload = np.array(workload_metric_data['rowlabels'])
# Target workload data
newest_result = Result.objects.get(pk=target_data['newest_result_id'])
X_target = target_data['X_matrix']
y_target = target_data['y_matrix']
rowlabels_target = np.array(target_data['rowlabels'])
if not np.array_equal(X_columnlabels, target_data['X_columnlabels']):
raise Exception(('The workload and target data should have '
'identical X columnlabels (sorted knob names)'))
if not np.array_equal(y_columnlabels, target_data['y_columnlabels']):
raise Exception(('The workload and target data should have '
'identical y columnlabels (sorted metric names)'))
# Filter Xs by top 10 ranked knobs
ranked_knobs = PipelineData.objects.get(
pipeline_run=latest_pipeline_run,
workload=mapped_workload,
task_type=PipelineTaskType.RANKED_KNOBS)
ranked_knobs = JSONUtil.loads(ranked_knobs.data)[:IMPORTANT_KNOB_NUMBER]
ranked_knob_idxs = [
i for i, cl in enumerate(X_columnlabels) if cl in ranked_knobs
]
X_workload = X_workload[:, ranked_knob_idxs]
X_target = X_target[:, ranked_knob_idxs]
X_columnlabels = X_columnlabels[ranked_knob_idxs]
# Filter ys by current target objective metric
target_objective = newest_result.session.target_objective
target_obj_idx = [
i for i, cl in enumerate(y_columnlabels) if cl == target_objective
]
if len(target_obj_idx) == 0:
raise Exception(('Could not find target objective in metrics '
'(target_obj={})').format(target_objective))
elif len(target_obj_idx) > 1:
raise Exception(
('Found {} instances of target objective in '
'metrics (target_obj={})').format(len(target_obj_idx),
target_objective))
metric_meta = db.target_objectives.get_metric_metadata(
newest_result.session.dbms.pk, newest_result.session.target_objective)
lessisbetter = metric_meta[
target_objective].improvement == db.target_objectives.LESS_IS_BETTER
y_workload = y_workload[:, target_obj_idx]
y_target = y_target[:, target_obj_idx]
y_columnlabels = y_columnlabels[target_obj_idx]
# Combine duplicate rows in the target/workload data (separately)
X_workload, y_workload, rowlabels_workload = DataUtil.combine_duplicate_rows(
X_workload, y_workload, rowlabels_workload)
X_target, y_target, rowlabels_target = DataUtil.combine_duplicate_rows(
X_target, y_target, rowlabels_target)
# Delete any rows that appear in both the workload data and the target
# data from the workload data
dups_filter = np.ones(X_workload.shape[0], dtype=bool)
target_row_tups = [tuple(row) for row in X_target]
for i, row in enumerate(X_workload):
if tuple(row) in target_row_tups:
dups_filter[i] = False
X_workload = X_workload[dups_filter, :]
y_workload = y_workload[dups_filter, :]
rowlabels_workload = rowlabels_workload[dups_filter]
# Combine target & workload Xs for preprocessing
X_matrix = np.vstack([X_target, X_workload])
# Dummy encode categorial variables
categorical_info = DataUtil.dummy_encoder_helper(X_columnlabels,
mapped_workload.dbms)
dummy_encoder = DummyEncoder(categorical_info['n_values'],
categorical_info['categorical_features'],
categorical_info['cat_columnlabels'],
categorical_info['noncat_columnlabels'])
X_matrix = dummy_encoder.fit_transform(X_matrix)
# below two variables are needed for correctly determing max/min on dummies
binary_index_set = set(categorical_info['binary_vars'])
total_dummies = dummy_encoder.total_dummies()
# Scale to N(0, 1)
X_scaler = StandardScaler()
X_scaled = X_scaler.fit_transform(X_matrix)
if y_target.shape[0] < 5: # FIXME
# FIXME (dva): if there are fewer than 5 target results so far
# then scale the y values (metrics) using the workload's
# y_scaler. I'm not sure if 5 is the right cutoff.
y_target_scaler = None
y_workload_scaler = StandardScaler()
y_matrix = np.vstack([y_target, y_workload])
y_scaled = y_workload_scaler.fit_transform(y_matrix)
else:
# FIXME (dva): otherwise try to compute a separate y_scaler for
# the target and scale them separately.
try:
y_target_scaler = StandardScaler()
y_workload_scaler = StandardScaler()
y_target_scaled = y_target_scaler.fit_transform(y_target)
y_workload_scaled = y_workload_scaler.fit_transform(y_workload)
y_scaled = np.vstack([y_target_scaled, y_workload_scaled])
except ValueError:
y_target_scaler = None
y_workload_scaler = StandardScaler()
y_scaled = y_workload_scaler.fit_transform(y_target)
# Set up constraint helper
constraint_helper = ParamConstraintHelper(
scaler=X_scaler,
encoder=dummy_encoder,
binary_vars=categorical_info['binary_vars'],
init_flip_prob=INIT_FLIP_PROB,
flip_prob_decay=FLIP_PROB_DECAY)
# FIXME (dva): check if these are good values for the ridge
# ridge = np.empty(X_scaled.shape[0])
# ridge[:X_target.shape[0]] = 0.01
# ridge[X_target.shape[0]:] = 0.1
# FIXME: we should generate more samples and use a smarter sampling
# technique
num_samples = NUM_SAMPLES
X_samples = np.empty((num_samples, X_scaled.shape[1]))
X_min = np.empty(X_scaled.shape[1])
X_max = np.empty(X_scaled.shape[1])
X_scaler_matrix = np.zeros([1, X_scaled.shape[1]])
session_knobs = SessionKnob.objects.get_knobs_for_session(
newest_result.session)
# Set min/max for knob values
for i in range(X_scaled.shape[1]):
if i < total_dummies or i in binary_index_set:
col_min = 0
col_max = 1
else:
col_min = X_scaled[:, i].min()
col_max = X_scaled[:, i].max()
for knob in session_knobs:
if X_columnlabels[i] == knob["name"]:
X_scaler_matrix[0][i] = knob["minval"]
col_min = X_scaler.transform(X_scaler_matrix)[0][i]
X_scaler_matrix[0][i] = knob["maxval"]
col_max = X_scaler.transform(X_scaler_matrix)[0][i]
X_min[i] = col_min
X_max[i] = col_max
X_samples[:, i] = np.random.rand(num_samples) * (col_max -
col_min) + col_min
# Maximize the throughput, moreisbetter
# Use gradient descent to minimize -throughput
if not lessisbetter:
y_scaled = -y_scaled
q = queue.PriorityQueue()
for x in range(0, y_scaled.shape[0]):
q.put((y_scaled[x][0], x))
i = 0
while i < TOP_NUM_CONFIG:
try:
item = q.get_nowait()
# Tensorflow get broken if we use the training data points as
# starting points for GPRGD. We add a small bias for the
# starting points. GPR_EPS default value is 0.001
# if the starting point is X_max, we minus a small bias to
# make sure it is within the range.
dist = sum(np.square(X_max - X_scaled[item[1]]))
if dist < 0.001:
X_samples = np.vstack(
(X_samples, X_scaled[item[1]] - abs(GPR_EPS)))
else:
X_samples = np.vstack(
(X_samples, X_scaled[item[1]] + abs(GPR_EPS)))
i = i + 1
except queue.Empty:
break
session = newest_result.session
res = None
if algorithm == AlgorithmType.DNN:
# neural network model
model_nn = NeuralNet(n_input=X_samples.shape[1],
batch_size=X_samples.shape[0],
explore_iters=DNN_EXPLORE_ITER,
noise_scale_begin=DNN_NOISE_SCALE_BEGIN,
noise_scale_end=DNN_NOISE_SCALE_END,
debug=DNN_DEBUG,
debug_interval=DNN_DEBUG_INTERVAL)
if session.dnn_model is not None:
model_nn.set_weights_bin(session.dnn_model)
model_nn.fit(X_scaled, y_scaled, fit_epochs=DNN_TRAIN_ITER)
res = model_nn.recommend(X_samples,
X_min,
X_max,
explore=DNN_EXPLORE,
recommend_epochs=MAX_ITER)
session.dnn_model = model_nn.get_weights_bin()
session.save()
elif algorithm == AlgorithmType.GPR:
# default gpr model
if USE_GPFLOW:
model_kwargs = {}
model_kwargs['model_learning_rate'] = HP_LEARNING_RATE
model_kwargs['model_maxiter'] = HP_MAX_ITER
opt_kwargs = {}
opt_kwargs['learning_rate'] = DEFAULT_LEARNING_RATE
opt_kwargs['maxiter'] = MAX_ITER
opt_kwargs['bounds'] = [X_min, X_max]
ucb_beta = 'get_beta_td'
opt_kwargs['ucb_beta'] = ucb.get_ucb_beta(ucb_beta,
scale=DEFAULT_UCB_SCALE,
t=i + 1.,
ndim=X_scaled.shape[1])
tf.reset_default_graph()
graph = tf.get_default_graph()
gpflow.reset_default_session(graph=graph)
m = gpr_models.create_model('BasicGP',
X=X_scaled,
y=y_scaled,
**model_kwargs)
res = tf_optimize(m.model, X_samples, **opt_kwargs)
else:
model = GPRGD(length_scale=DEFAULT_LENGTH_SCALE,
magnitude=DEFAULT_MAGNITUDE,
max_train_size=MAX_TRAIN_SIZE,
batch_size=BATCH_SIZE,
num_threads=NUM_THREADS,
learning_rate=DEFAULT_LEARNING_RATE,
epsilon=DEFAULT_EPSILON,
max_iter=MAX_ITER,
sigma_multiplier=DEFAULT_SIGMA_MULTIPLIER,
mu_multiplier=DEFAULT_MU_MULTIPLIER)
model.fit(X_scaled, y_scaled, X_min, X_max, ridge=DEFAULT_RIDGE)
res = model.predict(X_samples, constraint_helper=constraint_helper)
best_config_idx = np.argmin(res.minl.ravel())
best_config = res.minl_conf[best_config_idx, :]
best_config = X_scaler.inverse_transform(best_config)
# Decode one-hot encoding into categorical knobs
best_config = dummy_encoder.inverse_transform(best_config)
# Although we have max/min limits in the GPRGD training session, it may
# lose some precisions. e.g. 0.99..99 >= 1.0 may be True on the scaled data,
# when we inversely transform the scaled data, the different becomes much larger
# and cannot be ignored. Here we check the range on the original data
# directly, and make sure the recommended config lies within the range
X_min_inv = X_scaler.inverse_transform(X_min)
X_max_inv = X_scaler.inverse_transform(X_max)
best_config = np.minimum(best_config, X_max_inv)
best_config = np.maximum(best_config, X_min_inv)
conf_map = {k: best_config[i] for i, k in enumerate(X_columnlabels)}
conf_map_res = dict(status='good',
result_id=target_data['newest_result_id'],
recommendation=conf_map,
info='INFO: training data size is {}'.format(
X_scaled.shape[0]),
pipeline_run=latest_pipeline_run.pk)
LOG.debug('%s: Finished selecting the next config.\n\ndata=%s\n',
AlgorithmType.name(algorithm),
JSONUtil.dumps(conf_map_res, pprint=True))
return conf_map_res
def configuration_recommendation(recommendation_input):
target_data, algorithm = recommendation_input
LOG.info('configuration_recommendation called')
newest_result = Result.objects.get(pk=target_data['newest_result_id'])
session = newest_result.session
params = JSONUtil.loads(session.hyperparameters)
if target_data['bad'] is True:
target_data_res = create_and_save_recommendation(
recommended_knobs=target_data['config_recommend'],
result=newest_result,
status='bad',
info='WARNING: no training data, the config is generated randomly',
pipeline_run=target_data['pipeline_run'])
LOG.debug('%s: Skipping configuration recommendation.\n\ndata=%s\n',
AlgorithmType.name(algorithm),
JSONUtil.dumps(target_data, pprint=True))
return target_data_res
X_columnlabels, X_scaler, X_scaled, y_scaled, X_max, X_min,\
dummy_encoder, constraint_helper = combine_workload(target_data)
# FIXME: we should generate more samples and use a smarter sampling
# technique
num_samples = params['NUM_SAMPLES']
X_samples = np.empty((num_samples, X_scaled.shape[1]))
for i in range(X_scaled.shape[1]):
X_samples[:, i] = np.random.rand(num_samples) * (X_max[i] -
X_min[i]) + X_min[i]
q = queue.PriorityQueue()
for x in range(0, y_scaled.shape[0]):
q.put((y_scaled[x][0], x))
i = 0
while i < params['TOP_NUM_CONFIG']:
try:
item = q.get_nowait()
# Tensorflow get broken if we use the training data points as
# starting points for GPRGD. We add a small bias for the
# starting points. GPR_EPS default value is 0.001
# if the starting point is X_max, we minus a small bias to
# make sure it is within the range.
dist = sum(np.square(X_max - X_scaled[item[1]]))
if dist < 0.001:
X_samples = np.vstack(
(X_samples, X_scaled[item[1]] - abs(params['GPR_EPS'])))
else:
X_samples = np.vstack(
(X_samples, X_scaled[item[1]] + abs(params['GPR_EPS'])))
i = i + 1
except queue.Empty:
break
res = None
if algorithm == AlgorithmType.DNN:
# neural network model
model_nn = NeuralNet(n_input=X_samples.shape[1],
batch_size=X_samples.shape[0],
explore_iters=params['DNN_EXPLORE_ITER'],
noise_scale_begin=params['DNN_NOISE_SCALE_BEGIN'],
noise_scale_end=params['DNN_NOISE_SCALE_END'],
debug=params['DNN_DEBUG'],
debug_interval=params['DNN_DEBUG_INTERVAL'])
if session.dnn_model is not None:
model_nn.set_weights_bin(session.dnn_model)
model_nn.fit(X_scaled, y_scaled, fit_epochs=params['DNN_TRAIN_ITER'])
res = model_nn.recommend(X_samples,
X_min,
X_max,
explore=params['DNN_EXPLORE'],
recommend_epochs=params['DNN_GD_ITER'])
session.dnn_model = model_nn.get_weights_bin()
session.save()
elif algorithm == AlgorithmType.GPR:
# default gpr model
if params['GPR_USE_GPFLOW']:
model_kwargs = {}
model_kwargs['model_learning_rate'] = params[
'GPR_HP_LEARNING_RATE']
model_kwargs['model_maxiter'] = params['GPR_HP_MAX_ITER']
opt_kwargs = {}
opt_kwargs['learning_rate'] = params['GPR_LEARNING_RATE']
opt_kwargs['maxiter'] = params['GPR_MAX_ITER']
opt_kwargs['bounds'] = [X_min, X_max]
opt_kwargs['debug'] = params['GPR_DEBUG']
opt_kwargs['ucb_beta'] = ucb.get_ucb_beta(
params['GPR_UCB_BETA'],
scale=params['GPR_UCB_SCALE'],
t=i + 1.,
ndim=X_scaled.shape[1])
tf.reset_default_graph()
graph = tf.get_default_graph()
gpflow.reset_default_session(graph=graph)
m = gpr_models.create_model(params['GPR_MODEL_NAME'],
X=X_scaled,
y=y_scaled,
**model_kwargs)
res = tf_optimize(m.model, X_samples, **opt_kwargs)
else:
model = GPRGD(length_scale=params['GPR_LENGTH_SCALE'],
magnitude=params['GPR_MAGNITUDE'],
max_train_size=params['GPR_MAX_TRAIN_SIZE'],
batch_size=params['GPR_BATCH_SIZE'],
num_threads=params['TF_NUM_THREADS'],
learning_rate=params['GPR_LEARNING_RATE'],
epsilon=params['GPR_EPSILON'],
max_iter=params['GPR_MAX_ITER'],
sigma_multiplier=params['GPR_SIGMA_MULTIPLIER'],
mu_multiplier=params['GPR_MU_MULTIPLIER'],
ridge=params['GPR_RIDGE'])
model.fit(X_scaled, y_scaled, X_min, X_max)
res = model.predict(X_samples, constraint_helper=constraint_helper)
best_config_idx = np.argmin(res.minl.ravel())
best_config = res.minl_conf[best_config_idx, :]
best_config = X_scaler.inverse_transform(best_config)
if ENABLE_DUMMY_ENCODER:
# Decode one-hot encoding into categorical knobs
best_config = dummy_encoder.inverse_transform(best_config)
# Although we have max/min limits in the GPRGD training session, it may
# lose some precisions. e.g. 0.99..99 >= 1.0 may be True on the scaled data,
# when we inversely transform the scaled data, the different becomes much larger
# and cannot be ignored. Here we check the range on the original data
# directly, and make sure the recommended config lies within the range
X_min_inv = X_scaler.inverse_transform(X_min)
X_max_inv = X_scaler.inverse_transform(X_max)
best_config = np.minimum(best_config, X_max_inv)
best_config = np.maximum(best_config, X_min_inv)
conf_map = {k: best_config[i] for i, k in enumerate(X_columnlabels)}
conf_map_res = create_and_save_recommendation(
recommended_knobs=conf_map,
result=newest_result,
status='good',
info='INFO: training data size is {}'.format(X_scaled.shape[0]),
pipeline_run=target_data['pipeline_run'])
LOG.debug('%s: Finished selecting the next config.\n\ndata=%s\n',
AlgorithmType.name(algorithm),
JSONUtil.dumps(conf_map_res, pprint=True))
return conf_map_res
def reset_graph(self):
tf.reset_default_graph()
graph = tf.get_default_graph()
gpflow.reset_default_session(graph=graph)
def map_workload(map_workload_input):
start_ts = time.time()
target_data, algorithm = map_workload_input
if target_data['bad']:
assert target_data is not None
target_data['pipeline_run'] = None
LOG.debug('%s: Skipping workload mapping.\n\ndata=%s\n',
AlgorithmType.name(algorithm),
JSONUtil.dumps(target_data, pprint=True))
return target_data, algorithm
# Get the latest version of pipeline data that's been computed so far.
latest_pipeline_run = PipelineRun.objects.get_latest()
assert latest_pipeline_run is not None
target_data['pipeline_run'] = latest_pipeline_run.pk
newest_result = Result.objects.get(pk=target_data['newest_result_id'])
session = newest_result.session
params = JSONUtil.loads(session.hyperparameters)
target_workload = newest_result.workload
X_columnlabels = np.array(target_data['X_columnlabels'])
y_columnlabels = np.array(target_data['y_columnlabels'])
# Find all pipeline data belonging to the latest version with the same
# DBMS and hardware as the target
pipeline_data = PipelineData.objects.filter(
pipeline_run=latest_pipeline_run,
workload__dbms=target_workload.dbms,
workload__hardware=target_workload.hardware,
workload__project=target_workload.project)
# FIXME (dva): we should also compute the global (i.e., overall) ranked_knobs
# and pruned metrics but we just use those from the first workload for now
initialized = False
global_ranked_knobs = None
global_pruned_metrics = None
ranked_knob_idxs = None
pruned_metric_idxs = None
unique_workloads = pipeline_data.values_list('workload',
flat=True).distinct()
workload_data = {}
# Compute workload mapping data for each unique workload
for unique_workload in unique_workloads:
workload_obj = Workload.objects.get(pk=unique_workload)
wkld_results = Result.objects.filter(workload=workload_obj)
if wkld_results.exists() is False:
# delete the workload
workload_obj.delete()
continue
# Load knob & metric data for this workload
knob_data = load_data_helper(pipeline_data, unique_workload,
PipelineTaskType.KNOB_DATA)
knob_data["data"], knob_data["columnlabels"] = clean_knob_data(
knob_data["data"], knob_data["columnlabels"],
newest_result.session)
metric_data = load_data_helper(pipeline_data, unique_workload,
PipelineTaskType.METRIC_DATA)
X_matrix = np.array(knob_data["data"])
y_matrix = np.array(metric_data["data"])
rowlabels = np.array(knob_data["rowlabels"])
assert np.array_equal(rowlabels, metric_data["rowlabels"])
if not initialized:
# For now set ranked knobs & pruned metrics to be those computed
# for the first workload
global_ranked_knobs = load_data_helper(
pipeline_data, unique_workload, PipelineTaskType.RANKED_KNOBS
)[:params['IMPORTANT_KNOB_NUMBER']]
global_pruned_metrics = load_data_helper(
pipeline_data, unique_workload,
PipelineTaskType.PRUNED_METRICS)
ranked_knob_idxs = [
i for i in range(X_matrix.shape[1])
if X_columnlabels[i] in global_ranked_knobs
]
pruned_metric_idxs = [
i for i in range(y_matrix.shape[1])
if y_columnlabels[i] in global_pruned_metrics
]
# Filter X & y columnlabels by top ranked_knobs & pruned_metrics
X_columnlabels = X_columnlabels[ranked_knob_idxs]
y_columnlabels = y_columnlabels[pruned_metric_idxs]
initialized = True
# Filter X & y matrices by top ranked_knobs & pruned_metrics
X_matrix = X_matrix[:, ranked_knob_idxs]
y_matrix = y_matrix[:, pruned_metric_idxs]
# Combine duplicate rows (rows with same knob settings)
X_matrix, y_matrix, rowlabels = DataUtil.combine_duplicate_rows(
X_matrix, y_matrix, rowlabels)
workload_data[unique_workload] = {
'X_matrix': X_matrix,
'y_matrix': y_matrix,
'rowlabels': rowlabels,
}
if len(workload_data) == 0:
# The background task that aggregates the data has not finished running yet
target_data.update(mapped_workload=None, scores=None)
LOG.debug(
'%s: Skipping workload mapping because there is no parsed workload.\n',
AlgorithmType.name(algorithm))
return target_data, algorithm
# Stack all X & y matrices for preprocessing
Xs = np.vstack(
[entry['X_matrix'] for entry in list(workload_data.values())])
ys = np.vstack(
[entry['y_matrix'] for entry in list(workload_data.values())])
# Scale the X & y values, then compute the deciles for each column in y
X_scaler = StandardScaler(copy=False)
X_scaler.fit(Xs)
y_scaler = StandardScaler(copy=False)
y_scaler.fit_transform(ys)
y_binner = Bin(bin_start=1, axis=0)
y_binner.fit(ys)
del Xs
del ys
# Filter the target's X & y data by the ranked knobs & pruned metrics.
X_target = target_data['X_matrix'][:, ranked_knob_idxs]
y_target = target_data['y_matrix'][:, pruned_metric_idxs]
# Now standardize the target's data and bin it by the deciles we just
# calculated
X_target = X_scaler.transform(X_target)
y_target = y_scaler.transform(y_target)
y_target = y_binner.transform(y_target)
scores = {}
for workload_id, workload_entry in list(workload_data.items()):
predictions = np.empty_like(y_target)
X_workload = workload_entry['X_matrix']
X_scaled = X_scaler.transform(X_workload)
y_workload = workload_entry['y_matrix']
y_scaled = y_scaler.transform(y_workload)
for j, y_col in enumerate(y_scaled.T):
# Using this workload's data, train a Gaussian process model
# and then predict the performance of each metric for each of
# the knob configurations attempted so far by the target.
y_col = y_col.reshape(-1, 1)
if params['GPR_USE_GPFLOW']:
model_kwargs = {
'lengthscales': params['GPR_LENGTH_SCALE'],
'variance': params['GPR_MAGNITUDE'],
'noise_variance': params['GPR_RIDGE']
}
tf.reset_default_graph()
graph = tf.get_default_graph()
gpflow.reset_default_session(graph=graph)
m = gpr_models.create_model(params['GPR_MODEL_NAME'],
X=X_scaled,
y=y_col,
**model_kwargs)
gpr_result = gpflow_predict(m.model, X_target)
else:
model = GPRNP(length_scale=params['GPR_LENGTH_SCALE'],
magnitude=params['GPR_MAGNITUDE'],
max_train_size=params['GPR_MAX_TRAIN_SIZE'],
batch_size=params['GPR_BATCH_SIZE'])
model.fit(X_scaled, y_col, ridge=params['GPR_RIDGE'])
gpr_result = model.predict(X_target)
predictions[:, j] = gpr_result.ypreds.ravel()
# Bin each of the predicted metric columns by deciles and then
# compute the score (i.e., distance) between the target workload
# and each of the known workloads
predictions = y_binner.transform(predictions)
dists = np.sqrt(
np.sum(np.square(np.subtract(predictions, y_target)), axis=1))
scores[workload_id] = np.mean(dists)
# Find the best (minimum) score
best_score = np.inf
best_workload_id = None
best_workload_name = None
scores_info = {}
for workload_id, similarity_score in list(scores.items()):
workload_name = Workload.objects.get(pk=workload_id).name
if similarity_score < best_score:
best_score = similarity_score
best_workload_id = workload_id
best_workload_name = workload_name
scores_info[workload_id] = (workload_name, similarity_score)
target_data.update(mapped_workload=(best_workload_id, best_workload_name,
best_score),
scores=scores_info)
LOG.debug('%s: Finished mapping the workload.\n\ndata=%s\n',
AlgorithmType.name(algorithm),
JSONUtil.dumps(target_data, pprint=True))
save_execution_time(start_ts, "map_workload", newest_result)
return target_data, algorithm
def train_gpsig_classifier(dataset, num_levels=4, num_inducing=500, normalize_data=True, minibatch_size=50, max_len=400, increments=True, learn_weights=False,
num_lags=None, low_rank=False, val_split=None, test_split=None, experiment_idx=None, use_tensors=True, save_dir='./GPSig/'):
print('####################################')
print('Training dataset: {}'.format(dataset))
print('####################################')
## load data
X_train, y_train, X_val, y_val, X_test, y_test = load_dataset(dataset, val_split=val_split, test_split=test_split,
normalize_data=normalize_data, add_time=True, for_model='sig', max_len=max_len)
num_train, len_examples, num_features = X_train.shape
num_val = X_val.shape[0] if X_val is not None else None
num_test = X_test.shape[0]
num_classes = np.unique(y_train).size
with tf.Session(graph=tf.Graph(), config=tf.ConfigProto(gpu_options=tf.GPUOptions(allow_growth=True))) as sess:
## initialize inducing tensors and lengthsacles
if use_tensors:
Z_init = suggest_initial_inducing_tensors(X_train, num_levels, num_inducing, labels=y_train, increments=increments, num_lags=num_lags)
else:
Z_init = suggest_initial_inducing_sequences(X_train, num_inducing, num_levels+1, labels=y_train)
l_init = suggest_initial_lengthscales(X_train, num_samples=1000)
## reshape data into 2 axes format for gpflow
input_dim = len_examples * num_features
X_train = X_train.reshape([-1, input_dim])
X_val = X_val.reshape([-1, input_dim]) if X_val is not None else None
X_test = X_test.reshape([-1, input_dim])
## setup model
if use_tensors:
feat = gpsig.inducing_variables.InducingTensors(Z_init, num_levels=num_levels, increments=increments, learn_weights=learn_weights)
else:
feat = gpsig.inducing_variables.InducingSequences(Z_init, num_levels=num_levels, learn_weights=learn_weights)
k = gpsig.kernels.SignatureRBF(input_dim, num_levels=num_levels, num_features=num_features, lengthscales=l_init, num_lags=num_lags, low_rank=low_rank)
if num_classes == 2:
lik = gp.likelihoods.Bernoulli()
num_latent = 1
else:
lik = gp.likelihoods.MultiClass(num_classes)
num_latent = num_classes
m = gpsig.models.SVGP(X_train, y_train[:, None], kern=k, feat=feat, likelihood=lik, num_latent=num_latent,
minibatch_size=minibatch_size if minibatch_size < num_train else None, whiten=True)
## setup metrics
def batch_predict_y(m, X, batch_size=None):
num_iters = int(np.ceil(X.shape[0] / batch_size))
y_pred = np.zeros((X.shape[0]), dtype=np.float64)
for i in range(num_iters):
slice_batch = slice(i*batch_size, np.minimum((i+1)*batch_size, X.shape[0]))
X_batch = X[slice_batch]
pred_batch = m.predict_y(X_batch)[0]
if pred_batch.shape[1] == 1:
y_pred[slice_batch] = pred_batch.flatten() > 0.5
else:
y_pred[slice_batch] = np.argmax(pred_batch, axis=1)
return y_pred
def batch_predict_density(m, X, y, batch_size=None):
num_iters = int(np.ceil(X.shape[0] / batch_size))
y_nlpp = np.zeros((X.shape[0]), dtype=np.float64)
for i in range(num_iters):
slice_batch = slice(i*batch_size, np.minimum((i+1)*batch_size, X.shape[0]))
X_batch = X[slice_batch]
y_batch = y[slice_batch, None]
y_nlpp[slice_batch] = m.predict_density(X_batch, y_batch).flatten()
return y_nlpp
acc = lambda m, X, y: accuracy_score(y, batch_predict_y(m, X, batch_size=minibatch_size))
nlpp = lambda m, X, y: -np.mean(batch_predict_density(m, X, y, batch_size=minibatch_size))
val_acc = lambda m: acc(m, X_val, y_val)
val_nlpp = lambda m: nlpp(m, X_val, y_val)
test_acc = lambda m: acc(m, X_test, y_test)
test_nlpp = lambda m: nlpp(m, X_test, y_test)
val_scorers = [val_acc, val_nlpp] if X_val is not None else None
## train model
opt = gpsig.training.NadamOptimizer
num_iter_per_epoch = int(np.ceil(float(num_train) / minibatch_size))
### phase 1 - pre-train variational distribution
print_freq = np.minimum(num_iter_per_epoch, 5)
save_freq = np.minimum(num_iter_per_epoch, 50)
patience = np.maximum(500 * num_iter_per_epoch, 5000)
m.kern.set_trainable(False)
hist = gpsig.training.optimize(m, opt(1e-3), max_iter=patience, print_freq=print_freq, save_freq=save_freq,
val_scorer=val_scorers, save_best_params=X_val is not None, lower_is_better=True)
### phase 2 - train kernel (with sigma_i=sigma_j fixed) with early stopping
m.kern.set_trainable(True)
m.kern.variances.set_trainable(False)
hist = gpsig.training.optimize(m, opt(1e-3), max_iter=5000*num_iter_per_epoch, print_freq=print_freq, save_freq=save_freq, history=hist, # global_step=global_step,
val_scorer=val_scorers, save_best_params=X_val is not None, lower_is_better=True, patience=patience)
### restore best parameters
if 'best' in hist and 'params' in hist['best']: m.assign(hist['best']['params'])
### phase 3 - train with all kernel hyperparameters unfixed
m.kern.variances.set_trainable(True)
hist = gpsig.training.optimize(m, opt(1e-3), max_iter=5000*num_iter_per_epoch, print_freq=print_freq, save_freq=save_freq, history=hist, # global_step=global_step,
val_scorer=val_scorers, save_best_params=X_val is not None, lower_is_better=True, patience=patience)
### restore best parameters
if 'best' in hist and 'params' in hist['best']: m.assign(hist['best']['params'])
### evaluate on validation data
val_nlpp = val_nlpp(m)
val_acc = val_acc(m)
print('Val. nlpp: {:.4f}'.format(val_nlpp))
print('Val. accuracy: {:.4f}'.format(val_acc))
### phase 4 - fix kernel parameters and train on rest of data to assimilate into variational approximation
m.kern.set_trainable(False)
if val_split is not None:
X_train, y_train = np.concatenate((X_train, X_val), axis=0), np.concatenate((y_train, y_val), axis=0)
m.X, m.Y = X_train, y_train
num_train = X_train.shape[0]
m.num_data = num_train
hist = gpsig.training.optimize(m, opt(1e-3), max_iter=patience, print_freq=print_freq, save_freq=save_freq, history=hist)
## evaluate on test data
test_nlpp = nlpp(m, X_test, y_test)
test_acc = acc(m, X_test, y_test)
test_report = classification_report(y_test, batch_predict_y(m, X_test, batch_size=minibatch_size))
print('Test nlpp: {:.4f}'.format(test_nlpp))
print('Test accuracy: {:.4f}'.format(test_acc))
print(test_report)
## save results to file
hist['results'] = {}
hist['results']['val_acc'] = val_acc
hist['results']['val_nlpp'] = val_nlpp
hist['results']['test_nlpp'] = test_nlpp
hist['results']['test_acc'] = test_acc
hist['results']['test_report'] = test_report
if not os.path.isdir(save_dir):
os.makedirs(save_dir)
experiment_name = '{}'.format(dataset)
if experiment_idx is not None:
experiment_name += '_{}'.format(experiment_idx)
with open(os.path.join(save_dir, experiment_name + '.pkl'), 'wb') as f:
pickle.dump(hist, f)
with open(os.path.join(save_dir, experiment_name + '.txt'), 'w') as f:
f.write('Val. nlpp: {:.4f}\n'.format(val_nlpp))
f.write('Val. accuracy: {:.4f}\n'.format(val_acc))
f.write('Test nlpp: {:.4f}\n'.format(test_nlpp))
f.write('Test accuracy: {:.4f}\n'.format(test_acc))
f.write('Test report:\n')
f.write(test_report)
## clear memory manually
gp.reset_default_session()
tf.reset_default_graph()
import gc
gc.collect()
return
"Authorization": "NEED OANDA API KEY",
"Content-Type": "application/json"
}
res = requests.get(url + p, headers=h).json()
chartData[cp][tf] = res["candles"]
# In[26]:
for cp in currencyPairs:
for tf in timeFrames:
#Reset tensorflow session and pass to gpflow
tensorflow.reset_default_graph()
graph = tensorflow.get_default_graph()
gpflow.reset_default_session(graph=graph)
#Store candle data for a specific currency pair and timeframe
candles = chartData[cp][tf]
#Stores the median value of candle for Y axis and
#sequence between 0 and 1.10 for X
data = np.zeros(shape=(N, 2))
maxVal = 0.
minVal = 1000.
for c in candles:
median = 0.0
i = candles.index(c)
closePipVal = float(candles[i]["mid"]["c"])
def train_gpsigrnn_classifier(dataset,
num_hidden=128,
num_levels=4,
num_inducing=500,
normalize_data=True,
minibatch_size=50,
rnn_type='lstm',
use_dropout=True,
max_len=500,
increments=True,
learn_weights=False,
num_lags=None,
val_split=None,
test_split=None,
experiment_idx=None,
save_dir='./GPSigRNN/'):
num_lags = num_lags or 0
rnn_type = rnn_type.lower()
if rnn_type not in ['lstm', 'gru']:
raise ValueError('rnn_type should be \'LSTM\' or \'GRU\'')
print(
'##########################################################################'
)
print('Training dataset: {} | RNN = {}, num_hidden = {}, use_dropout = {}'.
format(dataset, rnn_type, num_hidden, int(use_dropout)))
print(
'##########################################################################'
)
## load data
X_train, y_train, X_val, y_val, X_test, y_test = load_dataset(
dataset,
val_split=val_split,
test_split=test_split,
max_len=max_len,
normalize_data=normalize_data,
add_time=True,
for_model='nn')
num_train, len_examples, num_features = X_train.shape
num_val = X_val.shape[0] if X_val is not None else None
num_test = X_test.shape[0]
num_classes = np.unique(y_train).size
with tf.Session(graph=tf.Graph(),
config=tf.ConfigProto(gpu_options=tf.GPUOptions(
allow_growth=True))) as sess:
## setup NN part of model
K.set_floatx('float64')
x_tens = K.placeholder(shape=(None, len_examples, num_features),
dtype=gp.settings.float_type)
y_tens = K.placeholder(shape=(None, 1), dtype=gp.settings.float_type)
masking_layer = keras.layers.Masking(mask_value=0.,
input_shape=(len_examples,
num_features))
recurrent_dropout = 0.05 if use_dropout else 0.
dropout = 0.25 if use_dropout else 0.
if rnn_type == 'lstm':
rnn_layer = keras.layers.LSTM(num_hidden,
recurrent_dropout=recurrent_dropout,
dropout=dropout,
return_sequences=True)
elif rnn_type == 'gru':
rnn_layer = keras.layers.GRU(num_hidden,
recurrent_dropout=recurrent_dropout,
dropout=dropout,
return_sequences=True)
# this is a hack to make reshape work with masking
identity_layer = keras.layers.Lambda(lambda x: x,
output_shape=lambda s: s)
gp_input_dim = len_examples * num_hidden
reshape_layer = keras.layers.Reshape((gp_input_dim, ))
fx_tens = reshape_layer(
identity_layer(rnn_layer(masking_layer(x_tens))))
nn_params = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, rnn_type)
## setup GP part of model
if num_classes == 2:
lik = gp.likelihoods.Bernoulli()
num_latent = 1
else:
lik = gp.likelihoods.MultiClass(num_classes)
num_latent = num_classes
# this is just temporary
Z_init = np.random.randn(int(num_levels * (num_levels + 1) / 2),
num_inducing, 2, num_hidden * (num_lags + 1))
feat = gpsig.inducing_variables.InducingTensors(
Z_init,
num_levels=num_levels,
increments=increments,
learn_weights=learn_weights)
k = gpsig.kernels.SignatureRBF(gp_input_dim,
num_levels=num_levels,
num_features=num_hidden,
num_lags=num_lags)
if num_classes == 2:
lik = gp.likelihoods.Bernoulli()
num_latent = 1
else:
lik = gp.likelihoods.MultiClass(num_classes)
num_latent = num_classes
m = gpsig.models.SVGP(fx_tens,
y_tens,
kern=k,
feat=feat,
likelihood=lik,
num_latent=num_latent,
minibatch_size=None,
num_data=num_train)
# some tensorflow magic
loss = -m.likelihood_tensor
fmean, fvar = m._build_predict(fx_tens)
ymean, yvar = m.likelihood.predict_mean_and_var(fmean, fvar)
lpd = m.likelihood.predict_density(fmean, fvar, y_tens)
K.set_session(sess)
sess.run(tf.variables_initializer(var_list=nn_params))
## setup metrics
def batch_predict_y(X, batch_size=None):
num_iters = int(np.ceil(X.shape[0] / batch_size))
y_pred = np.zeros((X.shape[0]), dtype=np.float64)
for i in range(num_iters):
slice_batch = slice(
i * batch_size, np.minimum((i + 1) * batch_size,
X.shape[0]))
X_batch = X[slice_batch]
pred_batch = sess.run(ymean, feed_dict={x_tens: X_batch})
if pred_batch.shape[1] == 1:
y_pred[slice_batch] = pred_batch.flatten() > 0.5
else:
y_pred[slice_batch] = np.argmax(pred_batch, axis=1)
return y_pred
def batch_predict_density(X, y, batch_size=None):
num_iters = int(np.ceil(X.shape[0] / batch_size))
y_nlpp = np.zeros((X.shape[0]), dtype=np.float64)
for i in range(num_iters):
slice_batch = slice(
i * batch_size, np.minimum((i + 1) * batch_size,
X.shape[0]))
X_batch = X[slice_batch]
y_batch = y[slice_batch, None]
y_nlpp[slice_batch] = sess.run(lpd,
feed_dict={
x_tens: X_batch,
y_tens: y_batch
}).flatten()
return y_nlpp
acc = lambda X, y: accuracy_score(
y, batch_predict_y(X, batch_size=minibatch_size))
nlpp = lambda X, y: -np.mean(
batch_predict_density(X, y, batch_size=minibatch_size))
val_acc = lambda: acc(X_val, y_val)
val_nlpp = lambda: nlpp(X_val, y_val)
test_acc = lambda: acc(X_test, y_test)
test_nlpp = lambda: nlpp(X_test, y_test)
## initalize inducing points with RNN-images of random data examples
fX_samples = sess.run(fx_tens,
feed_dict={
x_tens:
X_train[np.random.choice(num_train,
size=num_inducing)]
})
fX_samples = fX_samples.reshape([-1, len_examples, num_hidden])
Z_init = suggest_initial_inducing_tensors(fX_samples,
num_levels,
num_inducing,
increments=increments,
num_lags=num_lags)
m.feature.Z = Z_init
## initialize lengthscales parameter of RBF kernel
fX_samples = sess.run(fx_tens,
feed_dict={
x_tens:
X_train[np.random.choice(num_train,
size=(np.minimum(
1000,
num_train)),
replace=False)]
})
fX_samples = fX_samples.reshape([-1, len_examples, num_hidden])
l_init = suggest_initial_lengthscales(fX_samples, num_samples=1000)
m.kern.lengthscales = l_init
## train_model
minibatch_size = np.minimum(50, num_train)
### phase 1 - pre-train variational distribution
m.kern.set_trainable(False)
hist = fit_nn_with_gp_layer(X_train,
y_train,
m.trainable_tensors,
x_tens,
y_tens,
loss,
sess,
minibatch_size=minibatch_size,
max_epochs=500)
### phase 3 - train model with unfixed sigma_i
m.kern.variances.set_trainable(True)
var_list = nn_params + m.trainable_tensors
history = fit_nn_with_gp_layer(X_train,
y_train,
var_list,
x_tens,
y_tens,
loss,
sess,
minibatch_size=minibatch_size,
max_epochs=5000,
val_scores=[val_acc, val_nlpp],
patience=500,
lower_is_better=True,
history=hist)
### restore best parameters
for i, var in enumerate(var_list):
sess.run(var.assign(history['best']['params'][i]))
### evaluate on validation data
val_nlpp = val_nlpp()
val_acc = val_acc()
print('Val. nlpp: {:.4f}'.format(val_nlpp))
print('Val. accuracy: {:.4f}'.format(val_acc))
### phase 4 - fix NN and kernel params and train on rest of data to assimilate into variational approximation
#### re-merge validation data into training data
if val_split is not None:
X_train, y_train = np.concatenate(
(X_train, X_val), axis=0), np.concatenate((y_train, y_val),
axis=0)
num_train = X_train.shape[0]
m.num_data = num_train
#### train variational distribution
m.kern.set_trainable(False)
history = fit_nn_with_gp_layer(X_train,
y_train,
m.trainable_tensors,
x_tens,
y_tens,
loss,
sess,
minibatch_size=minibatch_size,
max_epochs=500,
history=hist)
## evaluate model
test_nlpp = test_nlpp()
test_acc = test_acc()
test_report = classification_report(
y_test, batch_predict_y(X_test, batch_size=minibatch_size))
print('Test nlpp: {:.4f}'.format(test_nlpp))
print('Test accuracy: {:.4f}'.format(test_acc))
print(test_report)
## save results to file
history['results'] = {}
history['results']['val_acc'] = val_acc
history['results']['val_nlpp'] = val_nlpp
history['results']['test_nlpp'] = test_nlpp
history['results']['test_acc'] = test_acc
history['results']['test_report'] = test_report
if not os.path.isdir(save_dir):
os.makedirs(save_dir)
experiment_name = '{}_H{}_D{}'.format(dataset, num_hidden,
int(use_dropout))
if experiment_idx is not None:
experiment_name += '_{}'.format(experiment_idx)
with open(os.path.join(save_dir, experiment_name + '.pkl'), 'wb') as f:
pickle.dump(history, f)
with open(os.path.join(save_dir, experiment_name + '.txt'), 'w') as f:
f.write('Val. nlpp: {:.4f}\n'.format(val_nlpp))
f.write('Val. accuracy: {:.4f}\n'.format(val_acc))
f.write('Test. nlpp: {:.4f}\n'.format(test_nlpp))
f.write('Test accuracy: {:.4f}\n'.format(test_acc))
f.write('Test report:\n')
f.write(test_report)
# clear memory manually
gp.reset_default_session()
tf.reset_default_graph()
K.clear_session()
import gc
gc.collect()
return
