def setUpClass(cls): super(TestGPRGP, cls).setUpClass() boston = datasets.load_boston() data = boston['data'] X_train = data[0:500] X_test = data[500:501] y_train = boston['target'][0:500].reshape(500, 1) X_min = np.min(X_train, 0) X_max = np.max(X_train, 0) random.seed(0) np.random.seed(0) tf.set_random_seed(0) model_kwargs = {} opt_kwargs = {} opt_kwargs['learning_rate'] = 0.01 opt_kwargs['maxiter'] = 10 opt_kwargs['bounds'] = [X_min, X_max] opt_kwargs['ucb_beta'] = 1.0 tf.reset_default_graph() graph = tf.get_default_graph() gpflow.reset_default_session(graph=graph) cls.m = gpr_models.create_model('BasicGP', X=X_train, y=y_train, **model_kwargs) cls.gpr_result = tf_optimize(cls.m.model, X_test, **opt_kwargs)
def use_gpu(id): os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" # The GPU id to use, usually either "0" or "1", "2' "3" "4" os.environ["CUDA_VISIBLE_DEVICES"] = str(id) # without using any gpu # os.environ["CUDA_VISIBLE_DEVICES"]="" config = tf.ConfigProto() config.gpu_options.allow_growth = True # less memeory gpflow.reset_default_session(config=config)
def setUpClass(cls): super(TestGPRGPFlow, cls).setUpClass() boston = datasets.load_boston() data = boston['data'] X_train = data[0:500] X_test = data[500:] y_train = boston['target'][0:500].reshape(500, 1) model_kwargs = {'lengthscales': 1, 'variance': 1, 'noise_variance': 1} tf.reset_default_graph() graph = tf.get_default_graph() gpflow.reset_default_session(graph=graph) cls.m = gpr_models.create_model('BasicGP', X=X_train, y=y_train, **model_kwargs) cls.gpr_result = gpflow_predict(cls.m.model, X_test)
def run_optimize(X, y, X_samples, model_name, opt_kwargs, model_kwargs): timer = TimerStruct() # Create model (this also optimizes the hyperparameters if that option is enabled timer.start() tf.reset_default_graph() graph = tf.get_default_graph() gpflow.reset_default_session(graph=graph) m = gpr_models.create_model(model_name, X=X, y=y, **model_kwargs) timer.stop() model_creation_sec = timer.elapsed_seconds LOG.info(m.model.as_pandas_table()) # Optimize the DBMS's configuration knobs timer.start() res = tf_optimize(m.model, X_samples, **opt_kwargs) timer.stop() config_optimize_sec = timer.elapsed_seconds return res.minl_conf, res.minl, m.get_model_parameters(), m.get_hyperparameters()
def test_different_sessions(self): with self.test_context(): demo = Demo() # Force initialization. with self.test_context() as session1: gpflow.reset_default_session() opt = self.optimizer() opt.minimize(demo, maxiter=1) # Mild initialization requirement: default changed session case. with self.test_context() as session2: self.assertFalse(session1 == session2) gpflow.reset_default_session() opt = self.optimizer() opt.minimize(demo, maxiter=1, initialize=False) # Mild initialization requirement: pass session case. with self.test_context() as session3: opt = self.optimizer() opt.minimize(demo, maxiter=1, session=session3, initialize=False)
def test_different_sessions(self): with self.test_context() as session: demo = Demo() # Force initialization. with self.test_context() as session1: gpflow.reset_default_session() opt = self.optimizer() demo.initialize(session1, force=True) opt.minimize(demo, maxiter=1) # Mild initialization requirement: default changed session case. with self.test_context() as session2: self.assertFalse(session1 == session2) gpflow.reset_default_session() opt = self.optimizer() opt.minimize(demo, maxiter=1, initialize=True) # Mild initialization requirement: pass session case. with self.test_context() as session3: opt = self.optimizer() opt.minimize(demo, maxiter=1, session=session3, initialize=True)
def test_external_variables_in_model(self): with self.test_context(): dfloat = gpflow.settings.float_type var1_init = np.array(1.0, dtype=dfloat) var2_init = np.array(2.0, dtype=dfloat) var3_init = np.array(3.0, dtype=dfloat) var1 = tf.get_variable('var1', initializer=var1_init, dtype=dfloat) var2 = tf.get_variable('var2', initializer=var2_init, dtype=dfloat) var3 = tf.get_variable('var3', initializer=var3_init, trainable=False, dtype=dfloat) opt = self.optimizer() # Just initialize variable, but do not use it in training with self.test_context() as session: gpflow.reset_default_session() demo1 = Demo(add_to_inits=[var1], add_to_trainables=[], name="demo1") opt.minimize(demo1, maxiter=10) self.assertEqual(session.run(var1), var1_init) with self.test_context() as session: gpflow.reset_default_session() with self.assertRaises(tf.errors.FailedPreconditionError): demo2 = Demo(add_to_inits=[], add_to_trainables=[var2], name="demo2") opt.minimize(demo2, maxiter=10) with self.test_context() as session: gpflow.reset_default_session() demo3 = Demo(add_to_inits=[var1, var2, var3], add_to_trainables=[var2, var3], name="demo3") opt.minimize(demo3, maxiter=10) self.assertEqual(session.run(var1), var1_init) self.assertFalse(session.run(var3) == var3_init) self.assertFalse(session.run(var2) == var2_init)
def test_external_variables_in_model(self): with self.test_context(): dfloat = gpflow.settings.float_type var1_init = np.array(1.0, dtype=dfloat) var2_init = np.array(2.0, dtype=dfloat) var3_init = np.array(3.0, dtype=dfloat) var1 = tf.get_variable('var1', initializer=var1_init, dtype=dfloat) var2 = tf.get_variable('var2', initializer=var2_init, dtype=dfloat) var3 = tf.get_variable('var3', initializer=var3_init, trainable=False, dtype=dfloat) opt = self.optimizer() # Just initialize variable, but do not use it in training with self.test_context() as session: gpflow.reset_default_session() demo1 = Demo(add_to_inits=[var1], add_to_trainables=[], name="demo1") opt.minimize(demo1, maxiter=10) self.assertEqual(session.run(var1), var1_init) with self.test_context() as session: gpflow.reset_default_session() with self.assertRaises(tf.errors.FailedPreconditionError): demo2 = Demo(add_to_inits=[], add_to_trainables=[var2], name="demo2") opt.minimize(demo2, maxiter=10) with self.test_context() as session: gpflow.reset_default_session() demo3 = Demo(add_to_inits=[var1, var2, var3], add_to_trainables=[var2, var3], name="demo3") opt.minimize(demo3, maxiter=10) self.assertEqual(session.run(var1), var1_init) self.assertFalse(session.run(var3) == var3_init) self.assertFalse(session.run(var2) == var2_init)
def configuration_recommendation(recommendation_input): target_data, algorithm = recommendation_input LOG.info('configuration_recommendation called') if target_data['bad'] is True: target_data_res = dict( status='bad', result_id=target_data['newest_result_id'], info='WARNING: no training data, the config is generated randomly', recommendation=target_data['config_recommend'], pipeline_run=target_data['pipeline_run']) LOG.debug('%s: Skipping configuration recommendation.\n\ndata=%s\n', AlgorithmType.name(algorithm), JSONUtil.dumps(target_data, pprint=True)) return target_data_res # Load mapped workload data mapped_workload_id = target_data['mapped_workload'][0] latest_pipeline_run = PipelineRun.objects.get( pk=target_data['pipeline_run']) mapped_workload = Workload.objects.get(pk=mapped_workload_id) workload_knob_data = PipelineData.objects.get( pipeline_run=latest_pipeline_run, workload=mapped_workload, task_type=PipelineTaskType.KNOB_DATA) workload_knob_data = JSONUtil.loads(workload_knob_data.data) workload_metric_data = PipelineData.objects.get( pipeline_run=latest_pipeline_run, workload=mapped_workload, task_type=PipelineTaskType.METRIC_DATA) workload_metric_data = JSONUtil.loads(workload_metric_data.data) newest_result = Result.objects.get(pk=target_data['newest_result_id']) cleaned_workload_knob_data = clean_knob_data( workload_knob_data["data"], workload_knob_data["columnlabels"], newest_result.session) X_workload = np.array(cleaned_workload_knob_data[0]) X_columnlabels = np.array(cleaned_workload_knob_data[1]) y_workload = np.array(workload_metric_data['data']) y_columnlabels = np.array(workload_metric_data['columnlabels']) rowlabels_workload = np.array(workload_metric_data['rowlabels']) # Target workload data newest_result = Result.objects.get(pk=target_data['newest_result_id']) X_target = target_data['X_matrix'] y_target = target_data['y_matrix'] rowlabels_target = np.array(target_data['rowlabels']) if not np.array_equal(X_columnlabels, target_data['X_columnlabels']): raise Exception(('The workload and target data should have ' 'identical X columnlabels (sorted knob names)')) if not np.array_equal(y_columnlabels, target_data['y_columnlabels']): raise Exception(('The workload and target data should have ' 'identical y columnlabels (sorted metric names)')) # Filter Xs by top 10 ranked knobs ranked_knobs = PipelineData.objects.get( pipeline_run=latest_pipeline_run, workload=mapped_workload, task_type=PipelineTaskType.RANKED_KNOBS) ranked_knobs = JSONUtil.loads(ranked_knobs.data)[:IMPORTANT_KNOB_NUMBER] ranked_knob_idxs = [ i for i, cl in enumerate(X_columnlabels) if cl in ranked_knobs ] X_workload = X_workload[:, ranked_knob_idxs] X_target = X_target[:, ranked_knob_idxs] X_columnlabels = X_columnlabels[ranked_knob_idxs] # Filter ys by current target objective metric target_objective = newest_result.session.target_objective target_obj_idx = [ i for i, cl in enumerate(y_columnlabels) if cl == target_objective ] if len(target_obj_idx) == 0: raise Exception(('Could not find target objective in metrics ' '(target_obj={})').format(target_objective)) elif len(target_obj_idx) > 1: raise Exception( ('Found {} instances of target objective in ' 'metrics (target_obj={})').format(len(target_obj_idx), target_objective)) metric_meta = db.target_objectives.get_metric_metadata( newest_result.session.dbms.pk, newest_result.session.target_objective) lessisbetter = metric_meta[ target_objective].improvement == db.target_objectives.LESS_IS_BETTER y_workload = y_workload[:, target_obj_idx] y_target = y_target[:, target_obj_idx] y_columnlabels = y_columnlabels[target_obj_idx] # Combine duplicate rows in the target/workload data (separately) X_workload, y_workload, rowlabels_workload = DataUtil.combine_duplicate_rows( X_workload, y_workload, rowlabels_workload) X_target, y_target, rowlabels_target = DataUtil.combine_duplicate_rows( X_target, y_target, rowlabels_target) # Delete any rows that appear in both the workload data and the target # data from the workload data dups_filter = np.ones(X_workload.shape[0], dtype=bool) target_row_tups = [tuple(row) for row in X_target] for i, row in enumerate(X_workload): if tuple(row) in target_row_tups: dups_filter[i] = False X_workload = X_workload[dups_filter, :] y_workload = y_workload[dups_filter, :] rowlabels_workload = rowlabels_workload[dups_filter] # Combine target & workload Xs for preprocessing X_matrix = np.vstack([X_target, X_workload]) # Dummy encode categorial variables categorical_info = DataUtil.dummy_encoder_helper(X_columnlabels, mapped_workload.dbms) dummy_encoder = DummyEncoder(categorical_info['n_values'], categorical_info['categorical_features'], categorical_info['cat_columnlabels'], categorical_info['noncat_columnlabels']) X_matrix = dummy_encoder.fit_transform(X_matrix) # below two variables are needed for correctly determing max/min on dummies binary_index_set = set(categorical_info['binary_vars']) total_dummies = dummy_encoder.total_dummies() # Scale to N(0, 1) X_scaler = StandardScaler() X_scaled = X_scaler.fit_transform(X_matrix) if y_target.shape[0] < 5: # FIXME # FIXME (dva): if there are fewer than 5 target results so far # then scale the y values (metrics) using the workload's # y_scaler. I'm not sure if 5 is the right cutoff. y_target_scaler = None y_workload_scaler = StandardScaler() y_matrix = np.vstack([y_target, y_workload]) y_scaled = y_workload_scaler.fit_transform(y_matrix) else: # FIXME (dva): otherwise try to compute a separate y_scaler for # the target and scale them separately. try: y_target_scaler = StandardScaler() y_workload_scaler = StandardScaler() y_target_scaled = y_target_scaler.fit_transform(y_target) y_workload_scaled = y_workload_scaler.fit_transform(y_workload) y_scaled = np.vstack([y_target_scaled, y_workload_scaled]) except ValueError: y_target_scaler = None y_workload_scaler = StandardScaler() y_scaled = y_workload_scaler.fit_transform(y_target) # Set up constraint helper constraint_helper = ParamConstraintHelper( scaler=X_scaler, encoder=dummy_encoder, binary_vars=categorical_info['binary_vars'], init_flip_prob=INIT_FLIP_PROB, flip_prob_decay=FLIP_PROB_DECAY) # FIXME (dva): check if these are good values for the ridge # ridge = np.empty(X_scaled.shape[0]) # ridge[:X_target.shape[0]] = 0.01 # ridge[X_target.shape[0]:] = 0.1 # FIXME: we should generate more samples and use a smarter sampling # technique num_samples = NUM_SAMPLES X_samples = np.empty((num_samples, X_scaled.shape[1])) X_min = np.empty(X_scaled.shape[1]) X_max = np.empty(X_scaled.shape[1]) X_scaler_matrix = np.zeros([1, X_scaled.shape[1]]) session_knobs = SessionKnob.objects.get_knobs_for_session( newest_result.session) # Set min/max for knob values for i in range(X_scaled.shape[1]): if i < total_dummies or i in binary_index_set: col_min = 0 col_max = 1 else: col_min = X_scaled[:, i].min() col_max = X_scaled[:, i].max() for knob in session_knobs: if X_columnlabels[i] == knob["name"]: X_scaler_matrix[0][i] = knob["minval"] col_min = X_scaler.transform(X_scaler_matrix)[0][i] X_scaler_matrix[0][i] = knob["maxval"] col_max = X_scaler.transform(X_scaler_matrix)[0][i] X_min[i] = col_min X_max[i] = col_max X_samples[:, i] = np.random.rand(num_samples) * (col_max - col_min) + col_min # Maximize the throughput, moreisbetter # Use gradient descent to minimize -throughput if not lessisbetter: y_scaled = -y_scaled q = queue.PriorityQueue() for x in range(0, y_scaled.shape[0]): q.put((y_scaled[x][0], x)) i = 0 while i < TOP_NUM_CONFIG: try: item = q.get_nowait() # Tensorflow get broken if we use the training data points as # starting points for GPRGD. We add a small bias for the # starting points. GPR_EPS default value is 0.001 # if the starting point is X_max, we minus a small bias to # make sure it is within the range. dist = sum(np.square(X_max - X_scaled[item[1]])) if dist < 0.001: X_samples = np.vstack( (X_samples, X_scaled[item[1]] - abs(GPR_EPS))) else: X_samples = np.vstack( (X_samples, X_scaled[item[1]] + abs(GPR_EPS))) i = i + 1 except queue.Empty: break session = newest_result.session res = None if algorithm == AlgorithmType.DNN: # neural network model model_nn = NeuralNet(n_input=X_samples.shape[1], batch_size=X_samples.shape[0], explore_iters=DNN_EXPLORE_ITER, noise_scale_begin=DNN_NOISE_SCALE_BEGIN, noise_scale_end=DNN_NOISE_SCALE_END, debug=DNN_DEBUG, debug_interval=DNN_DEBUG_INTERVAL) if session.dnn_model is not None: model_nn.set_weights_bin(session.dnn_model) model_nn.fit(X_scaled, y_scaled, fit_epochs=DNN_TRAIN_ITER) res = model_nn.recommend(X_samples, X_min, X_max, explore=DNN_EXPLORE, recommend_epochs=MAX_ITER) session.dnn_model = model_nn.get_weights_bin() session.save() elif algorithm == AlgorithmType.GPR: # default gpr model if USE_GPFLOW: model_kwargs = {} model_kwargs['model_learning_rate'] = HP_LEARNING_RATE model_kwargs['model_maxiter'] = HP_MAX_ITER opt_kwargs = {} opt_kwargs['learning_rate'] = DEFAULT_LEARNING_RATE opt_kwargs['maxiter'] = MAX_ITER opt_kwargs['bounds'] = [X_min, X_max] ucb_beta = 'get_beta_td' opt_kwargs['ucb_beta'] = ucb.get_ucb_beta(ucb_beta, scale=DEFAULT_UCB_SCALE, t=i + 1., ndim=X_scaled.shape[1]) tf.reset_default_graph() graph = tf.get_default_graph() gpflow.reset_default_session(graph=graph) m = gpr_models.create_model('BasicGP', X=X_scaled, y=y_scaled, **model_kwargs) res = tf_optimize(m.model, X_samples, **opt_kwargs) else: model = GPRGD(length_scale=DEFAULT_LENGTH_SCALE, magnitude=DEFAULT_MAGNITUDE, max_train_size=MAX_TRAIN_SIZE, batch_size=BATCH_SIZE, num_threads=NUM_THREADS, learning_rate=DEFAULT_LEARNING_RATE, epsilon=DEFAULT_EPSILON, max_iter=MAX_ITER, sigma_multiplier=DEFAULT_SIGMA_MULTIPLIER, mu_multiplier=DEFAULT_MU_MULTIPLIER) model.fit(X_scaled, y_scaled, X_min, X_max, ridge=DEFAULT_RIDGE) res = model.predict(X_samples, constraint_helper=constraint_helper) best_config_idx = np.argmin(res.minl.ravel()) best_config = res.minl_conf[best_config_idx, :] best_config = X_scaler.inverse_transform(best_config) # Decode one-hot encoding into categorical knobs best_config = dummy_encoder.inverse_transform(best_config) # Although we have max/min limits in the GPRGD training session, it may # lose some precisions. e.g. 0.99..99 >= 1.0 may be True on the scaled data, # when we inversely transform the scaled data, the different becomes much larger # and cannot be ignored. Here we check the range on the original data # directly, and make sure the recommended config lies within the range X_min_inv = X_scaler.inverse_transform(X_min) X_max_inv = X_scaler.inverse_transform(X_max) best_config = np.minimum(best_config, X_max_inv) best_config = np.maximum(best_config, X_min_inv) conf_map = {k: best_config[i] for i, k in enumerate(X_columnlabels)} conf_map_res = dict(status='good', result_id=target_data['newest_result_id'], recommendation=conf_map, info='INFO: training data size is {}'.format( X_scaled.shape[0]), pipeline_run=latest_pipeline_run.pk) LOG.debug('%s: Finished selecting the next config.\n\ndata=%s\n', AlgorithmType.name(algorithm), JSONUtil.dumps(conf_map_res, pprint=True)) return conf_map_res
def configuration_recommendation(recommendation_input): target_data, algorithm = recommendation_input LOG.info('configuration_recommendation called') newest_result = Result.objects.get(pk=target_data['newest_result_id']) session = newest_result.session params = JSONUtil.loads(session.hyperparameters) if target_data['bad'] is True: target_data_res = create_and_save_recommendation( recommended_knobs=target_data['config_recommend'], result=newest_result, status='bad', info='WARNING: no training data, the config is generated randomly', pipeline_run=target_data['pipeline_run']) LOG.debug('%s: Skipping configuration recommendation.\n\ndata=%s\n', AlgorithmType.name(algorithm), JSONUtil.dumps(target_data, pprint=True)) return target_data_res X_columnlabels, X_scaler, X_scaled, y_scaled, X_max, X_min,\ dummy_encoder, constraint_helper = combine_workload(target_data) # FIXME: we should generate more samples and use a smarter sampling # technique num_samples = params['NUM_SAMPLES'] X_samples = np.empty((num_samples, X_scaled.shape[1])) for i in range(X_scaled.shape[1]): X_samples[:, i] = np.random.rand(num_samples) * (X_max[i] - X_min[i]) + X_min[i] q = queue.PriorityQueue() for x in range(0, y_scaled.shape[0]): q.put((y_scaled[x][0], x)) i = 0 while i < params['TOP_NUM_CONFIG']: try: item = q.get_nowait() # Tensorflow get broken if we use the training data points as # starting points for GPRGD. We add a small bias for the # starting points. GPR_EPS default value is 0.001 # if the starting point is X_max, we minus a small bias to # make sure it is within the range. dist = sum(np.square(X_max - X_scaled[item[1]])) if dist < 0.001: X_samples = np.vstack( (X_samples, X_scaled[item[1]] - abs(params['GPR_EPS']))) else: X_samples = np.vstack( (X_samples, X_scaled[item[1]] + abs(params['GPR_EPS']))) i = i + 1 except queue.Empty: break res = None if algorithm == AlgorithmType.DNN: # neural network model model_nn = NeuralNet(n_input=X_samples.shape[1], batch_size=X_samples.shape[0], explore_iters=params['DNN_EXPLORE_ITER'], noise_scale_begin=params['DNN_NOISE_SCALE_BEGIN'], noise_scale_end=params['DNN_NOISE_SCALE_END'], debug=params['DNN_DEBUG'], debug_interval=params['DNN_DEBUG_INTERVAL']) if session.dnn_model is not None: model_nn.set_weights_bin(session.dnn_model) model_nn.fit(X_scaled, y_scaled, fit_epochs=params['DNN_TRAIN_ITER']) res = model_nn.recommend(X_samples, X_min, X_max, explore=params['DNN_EXPLORE'], recommend_epochs=params['DNN_GD_ITER']) session.dnn_model = model_nn.get_weights_bin() session.save() elif algorithm == AlgorithmType.GPR: # default gpr model if params['GPR_USE_GPFLOW']: model_kwargs = {} model_kwargs['model_learning_rate'] = params[ 'GPR_HP_LEARNING_RATE'] model_kwargs['model_maxiter'] = params['GPR_HP_MAX_ITER'] opt_kwargs = {} opt_kwargs['learning_rate'] = params['GPR_LEARNING_RATE'] opt_kwargs['maxiter'] = params['GPR_MAX_ITER'] opt_kwargs['bounds'] = [X_min, X_max] opt_kwargs['debug'] = params['GPR_DEBUG'] opt_kwargs['ucb_beta'] = ucb.get_ucb_beta( params['GPR_UCB_BETA'], scale=params['GPR_UCB_SCALE'], t=i + 1., ndim=X_scaled.shape[1]) tf.reset_default_graph() graph = tf.get_default_graph() gpflow.reset_default_session(graph=graph) m = gpr_models.create_model(params['GPR_MODEL_NAME'], X=X_scaled, y=y_scaled, **model_kwargs) res = tf_optimize(m.model, X_samples, **opt_kwargs) else: model = GPRGD(length_scale=params['GPR_LENGTH_SCALE'], magnitude=params['GPR_MAGNITUDE'], max_train_size=params['GPR_MAX_TRAIN_SIZE'], batch_size=params['GPR_BATCH_SIZE'], num_threads=params['TF_NUM_THREADS'], learning_rate=params['GPR_LEARNING_RATE'], epsilon=params['GPR_EPSILON'], max_iter=params['GPR_MAX_ITER'], sigma_multiplier=params['GPR_SIGMA_MULTIPLIER'], mu_multiplier=params['GPR_MU_MULTIPLIER'], ridge=params['GPR_RIDGE']) model.fit(X_scaled, y_scaled, X_min, X_max) res = model.predict(X_samples, constraint_helper=constraint_helper) best_config_idx = np.argmin(res.minl.ravel()) best_config = res.minl_conf[best_config_idx, :] best_config = X_scaler.inverse_transform(best_config) if ENABLE_DUMMY_ENCODER: # Decode one-hot encoding into categorical knobs best_config = dummy_encoder.inverse_transform(best_config) # Although we have max/min limits in the GPRGD training session, it may # lose some precisions. e.g. 0.99..99 >= 1.0 may be True on the scaled data, # when we inversely transform the scaled data, the different becomes much larger # and cannot be ignored. Here we check the range on the original data # directly, and make sure the recommended config lies within the range X_min_inv = X_scaler.inverse_transform(X_min) X_max_inv = X_scaler.inverse_transform(X_max) best_config = np.minimum(best_config, X_max_inv) best_config = np.maximum(best_config, X_min_inv) conf_map = {k: best_config[i] for i, k in enumerate(X_columnlabels)} conf_map_res = create_and_save_recommendation( recommended_knobs=conf_map, result=newest_result, status='good', info='INFO: training data size is {}'.format(X_scaled.shape[0]), pipeline_run=target_data['pipeline_run']) LOG.debug('%s: Finished selecting the next config.\n\ndata=%s\n', AlgorithmType.name(algorithm), JSONUtil.dumps(conf_map_res, pprint=True)) return conf_map_res
def reset_graph(self): tf.reset_default_graph() graph = tf.get_default_graph() gpflow.reset_default_session(graph=graph)
def map_workload(map_workload_input): start_ts = time.time() target_data, algorithm = map_workload_input if target_data['bad']: assert target_data is not None target_data['pipeline_run'] = None LOG.debug('%s: Skipping workload mapping.\n\ndata=%s\n', AlgorithmType.name(algorithm), JSONUtil.dumps(target_data, pprint=True)) return target_data, algorithm # Get the latest version of pipeline data that's been computed so far. latest_pipeline_run = PipelineRun.objects.get_latest() assert latest_pipeline_run is not None target_data['pipeline_run'] = latest_pipeline_run.pk newest_result = Result.objects.get(pk=target_data['newest_result_id']) session = newest_result.session params = JSONUtil.loads(session.hyperparameters) target_workload = newest_result.workload X_columnlabels = np.array(target_data['X_columnlabels']) y_columnlabels = np.array(target_data['y_columnlabels']) # Find all pipeline data belonging to the latest version with the same # DBMS and hardware as the target pipeline_data = PipelineData.objects.filter( pipeline_run=latest_pipeline_run, workload__dbms=target_workload.dbms, workload__hardware=target_workload.hardware, workload__project=target_workload.project) # FIXME (dva): we should also compute the global (i.e., overall) ranked_knobs # and pruned metrics but we just use those from the first workload for now initialized = False global_ranked_knobs = None global_pruned_metrics = None ranked_knob_idxs = None pruned_metric_idxs = None unique_workloads = pipeline_data.values_list('workload', flat=True).distinct() workload_data = {} # Compute workload mapping data for each unique workload for unique_workload in unique_workloads: workload_obj = Workload.objects.get(pk=unique_workload) wkld_results = Result.objects.filter(workload=workload_obj) if wkld_results.exists() is False: # delete the workload workload_obj.delete() continue # Load knob & metric data for this workload knob_data = load_data_helper(pipeline_data, unique_workload, PipelineTaskType.KNOB_DATA) knob_data["data"], knob_data["columnlabels"] = clean_knob_data( knob_data["data"], knob_data["columnlabels"], newest_result.session) metric_data = load_data_helper(pipeline_data, unique_workload, PipelineTaskType.METRIC_DATA) X_matrix = np.array(knob_data["data"]) y_matrix = np.array(metric_data["data"]) rowlabels = np.array(knob_data["rowlabels"]) assert np.array_equal(rowlabels, metric_data["rowlabels"]) if not initialized: # For now set ranked knobs & pruned metrics to be those computed # for the first workload global_ranked_knobs = load_data_helper( pipeline_data, unique_workload, PipelineTaskType.RANKED_KNOBS )[:params['IMPORTANT_KNOB_NUMBER']] global_pruned_metrics = load_data_helper( pipeline_data, unique_workload, PipelineTaskType.PRUNED_METRICS) ranked_knob_idxs = [ i for i in range(X_matrix.shape[1]) if X_columnlabels[i] in global_ranked_knobs ] pruned_metric_idxs = [ i for i in range(y_matrix.shape[1]) if y_columnlabels[i] in global_pruned_metrics ] # Filter X & y columnlabels by top ranked_knobs & pruned_metrics X_columnlabels = X_columnlabels[ranked_knob_idxs] y_columnlabels = y_columnlabels[pruned_metric_idxs] initialized = True # Filter X & y matrices by top ranked_knobs & pruned_metrics X_matrix = X_matrix[:, ranked_knob_idxs] y_matrix = y_matrix[:, pruned_metric_idxs] # Combine duplicate rows (rows with same knob settings) X_matrix, y_matrix, rowlabels = DataUtil.combine_duplicate_rows( X_matrix, y_matrix, rowlabels) workload_data[unique_workload] = { 'X_matrix': X_matrix, 'y_matrix': y_matrix, 'rowlabels': rowlabels, } if len(workload_data) == 0: # The background task that aggregates the data has not finished running yet target_data.update(mapped_workload=None, scores=None) LOG.debug( '%s: Skipping workload mapping because there is no parsed workload.\n', AlgorithmType.name(algorithm)) return target_data, algorithm # Stack all X & y matrices for preprocessing Xs = np.vstack( [entry['X_matrix'] for entry in list(workload_data.values())]) ys = np.vstack( [entry['y_matrix'] for entry in list(workload_data.values())]) # Scale the X & y values, then compute the deciles for each column in y X_scaler = StandardScaler(copy=False) X_scaler.fit(Xs) y_scaler = StandardScaler(copy=False) y_scaler.fit_transform(ys) y_binner = Bin(bin_start=1, axis=0) y_binner.fit(ys) del Xs del ys # Filter the target's X & y data by the ranked knobs & pruned metrics. X_target = target_data['X_matrix'][:, ranked_knob_idxs] y_target = target_data['y_matrix'][:, pruned_metric_idxs] # Now standardize the target's data and bin it by the deciles we just # calculated X_target = X_scaler.transform(X_target) y_target = y_scaler.transform(y_target) y_target = y_binner.transform(y_target) scores = {} for workload_id, workload_entry in list(workload_data.items()): predictions = np.empty_like(y_target) X_workload = workload_entry['X_matrix'] X_scaled = X_scaler.transform(X_workload) y_workload = workload_entry['y_matrix'] y_scaled = y_scaler.transform(y_workload) for j, y_col in enumerate(y_scaled.T): # Using this workload's data, train a Gaussian process model # and then predict the performance of each metric for each of # the knob configurations attempted so far by the target. y_col = y_col.reshape(-1, 1) if params['GPR_USE_GPFLOW']: model_kwargs = { 'lengthscales': params['GPR_LENGTH_SCALE'], 'variance': params['GPR_MAGNITUDE'], 'noise_variance': params['GPR_RIDGE'] } tf.reset_default_graph() graph = tf.get_default_graph() gpflow.reset_default_session(graph=graph) m = gpr_models.create_model(params['GPR_MODEL_NAME'], X=X_scaled, y=y_col, **model_kwargs) gpr_result = gpflow_predict(m.model, X_target) else: model = GPRNP(length_scale=params['GPR_LENGTH_SCALE'], magnitude=params['GPR_MAGNITUDE'], max_train_size=params['GPR_MAX_TRAIN_SIZE'], batch_size=params['GPR_BATCH_SIZE']) model.fit(X_scaled, y_col, ridge=params['GPR_RIDGE']) gpr_result = model.predict(X_target) predictions[:, j] = gpr_result.ypreds.ravel() # Bin each of the predicted metric columns by deciles and then # compute the score (i.e., distance) between the target workload # and each of the known workloads predictions = y_binner.transform(predictions) dists = np.sqrt( np.sum(np.square(np.subtract(predictions, y_target)), axis=1)) scores[workload_id] = np.mean(dists) # Find the best (minimum) score best_score = np.inf best_workload_id = None best_workload_name = None scores_info = {} for workload_id, similarity_score in list(scores.items()): workload_name = Workload.objects.get(pk=workload_id).name if similarity_score < best_score: best_score = similarity_score best_workload_id = workload_id best_workload_name = workload_name scores_info[workload_id] = (workload_name, similarity_score) target_data.update(mapped_workload=(best_workload_id, best_workload_name, best_score), scores=scores_info) LOG.debug('%s: Finished mapping the workload.\n\ndata=%s\n', AlgorithmType.name(algorithm), JSONUtil.dumps(target_data, pprint=True)) save_execution_time(start_ts, "map_workload", newest_result) return target_data, algorithm
def train_gpsig_classifier(dataset, num_levels=4, num_inducing=500, normalize_data=True, minibatch_size=50, max_len=400, increments=True, learn_weights=False, num_lags=None, low_rank=False, val_split=None, test_split=None, experiment_idx=None, use_tensors=True, save_dir='./GPSig/'): print('####################################') print('Training dataset: {}'.format(dataset)) print('####################################') ## load data X_train, y_train, X_val, y_val, X_test, y_test = load_dataset(dataset, val_split=val_split, test_split=test_split, normalize_data=normalize_data, add_time=True, for_model='sig', max_len=max_len) num_train, len_examples, num_features = X_train.shape num_val = X_val.shape[0] if X_val is not None else None num_test = X_test.shape[0] num_classes = np.unique(y_train).size with tf.Session(graph=tf.Graph(), config=tf.ConfigProto(gpu_options=tf.GPUOptions(allow_growth=True))) as sess: ## initialize inducing tensors and lengthsacles if use_tensors: Z_init = suggest_initial_inducing_tensors(X_train, num_levels, num_inducing, labels=y_train, increments=increments, num_lags=num_lags) else: Z_init = suggest_initial_inducing_sequences(X_train, num_inducing, num_levels+1, labels=y_train) l_init = suggest_initial_lengthscales(X_train, num_samples=1000) ## reshape data into 2 axes format for gpflow input_dim = len_examples * num_features X_train = X_train.reshape([-1, input_dim]) X_val = X_val.reshape([-1, input_dim]) if X_val is not None else None X_test = X_test.reshape([-1, input_dim]) ## setup model if use_tensors: feat = gpsig.inducing_variables.InducingTensors(Z_init, num_levels=num_levels, increments=increments, learn_weights=learn_weights) else: feat = gpsig.inducing_variables.InducingSequences(Z_init, num_levels=num_levels, learn_weights=learn_weights) k = gpsig.kernels.SignatureRBF(input_dim, num_levels=num_levels, num_features=num_features, lengthscales=l_init, num_lags=num_lags, low_rank=low_rank) if num_classes == 2: lik = gp.likelihoods.Bernoulli() num_latent = 1 else: lik = gp.likelihoods.MultiClass(num_classes) num_latent = num_classes m = gpsig.models.SVGP(X_train, y_train[:, None], kern=k, feat=feat, likelihood=lik, num_latent=num_latent, minibatch_size=minibatch_size if minibatch_size < num_train else None, whiten=True) ## setup metrics def batch_predict_y(m, X, batch_size=None): num_iters = int(np.ceil(X.shape[0] / batch_size)) y_pred = np.zeros((X.shape[0]), dtype=np.float64) for i in range(num_iters): slice_batch = slice(i*batch_size, np.minimum((i+1)*batch_size, X.shape[0])) X_batch = X[slice_batch] pred_batch = m.predict_y(X_batch)[0] if pred_batch.shape[1] == 1: y_pred[slice_batch] = pred_batch.flatten() > 0.5 else: y_pred[slice_batch] = np.argmax(pred_batch, axis=1) return y_pred def batch_predict_density(m, X, y, batch_size=None): num_iters = int(np.ceil(X.shape[0] / batch_size)) y_nlpp = np.zeros((X.shape[0]), dtype=np.float64) for i in range(num_iters): slice_batch = slice(i*batch_size, np.minimum((i+1)*batch_size, X.shape[0])) X_batch = X[slice_batch] y_batch = y[slice_batch, None] y_nlpp[slice_batch] = m.predict_density(X_batch, y_batch).flatten() return y_nlpp acc = lambda m, X, y: accuracy_score(y, batch_predict_y(m, X, batch_size=minibatch_size)) nlpp = lambda m, X, y: -np.mean(batch_predict_density(m, X, y, batch_size=minibatch_size)) val_acc = lambda m: acc(m, X_val, y_val) val_nlpp = lambda m: nlpp(m, X_val, y_val) test_acc = lambda m: acc(m, X_test, y_test) test_nlpp = lambda m: nlpp(m, X_test, y_test) val_scorers = [val_acc, val_nlpp] if X_val is not None else None ## train model opt = gpsig.training.NadamOptimizer num_iter_per_epoch = int(np.ceil(float(num_train) / minibatch_size)) ### phase 1 - pre-train variational distribution print_freq = np.minimum(num_iter_per_epoch, 5) save_freq = np.minimum(num_iter_per_epoch, 50) patience = np.maximum(500 * num_iter_per_epoch, 5000) m.kern.set_trainable(False) hist = gpsig.training.optimize(m, opt(1e-3), max_iter=patience, print_freq=print_freq, save_freq=save_freq, val_scorer=val_scorers, save_best_params=X_val is not None, lower_is_better=True) ### phase 2 - train kernel (with sigma_i=sigma_j fixed) with early stopping m.kern.set_trainable(True) m.kern.variances.set_trainable(False) hist = gpsig.training.optimize(m, opt(1e-3), max_iter=5000*num_iter_per_epoch, print_freq=print_freq, save_freq=save_freq, history=hist, # global_step=global_step, val_scorer=val_scorers, save_best_params=X_val is not None, lower_is_better=True, patience=patience) ### restore best parameters if 'best' in hist and 'params' in hist['best']: m.assign(hist['best']['params']) ### phase 3 - train with all kernel hyperparameters unfixed m.kern.variances.set_trainable(True) hist = gpsig.training.optimize(m, opt(1e-3), max_iter=5000*num_iter_per_epoch, print_freq=print_freq, save_freq=save_freq, history=hist, # global_step=global_step, val_scorer=val_scorers, save_best_params=X_val is not None, lower_is_better=True, patience=patience) ### restore best parameters if 'best' in hist and 'params' in hist['best']: m.assign(hist['best']['params']) ### evaluate on validation data val_nlpp = val_nlpp(m) val_acc = val_acc(m) print('Val. nlpp: {:.4f}'.format(val_nlpp)) print('Val. accuracy: {:.4f}'.format(val_acc)) ### phase 4 - fix kernel parameters and train on rest of data to assimilate into variational approximation m.kern.set_trainable(False) if val_split is not None: X_train, y_train = np.concatenate((X_train, X_val), axis=0), np.concatenate((y_train, y_val), axis=0) m.X, m.Y = X_train, y_train num_train = X_train.shape[0] m.num_data = num_train hist = gpsig.training.optimize(m, opt(1e-3), max_iter=patience, print_freq=print_freq, save_freq=save_freq, history=hist) ## evaluate on test data test_nlpp = nlpp(m, X_test, y_test) test_acc = acc(m, X_test, y_test) test_report = classification_report(y_test, batch_predict_y(m, X_test, batch_size=minibatch_size)) print('Test nlpp: {:.4f}'.format(test_nlpp)) print('Test accuracy: {:.4f}'.format(test_acc)) print(test_report) ## save results to file hist['results'] = {} hist['results']['val_acc'] = val_acc hist['results']['val_nlpp'] = val_nlpp hist['results']['test_nlpp'] = test_nlpp hist['results']['test_acc'] = test_acc hist['results']['test_report'] = test_report if not os.path.isdir(save_dir): os.makedirs(save_dir) experiment_name = '{}'.format(dataset) if experiment_idx is not None: experiment_name += '_{}'.format(experiment_idx) with open(os.path.join(save_dir, experiment_name + '.pkl'), 'wb') as f: pickle.dump(hist, f) with open(os.path.join(save_dir, experiment_name + '.txt'), 'w') as f: f.write('Val. nlpp: {:.4f}\n'.format(val_nlpp)) f.write('Val. accuracy: {:.4f}\n'.format(val_acc)) f.write('Test nlpp: {:.4f}\n'.format(test_nlpp)) f.write('Test accuracy: {:.4f}\n'.format(test_acc)) f.write('Test report:\n') f.write(test_report) ## clear memory manually gp.reset_default_session() tf.reset_default_graph() import gc gc.collect() return
"Authorization": "NEED OANDA API KEY", "Content-Type": "application/json" } res = requests.get(url + p, headers=h).json() chartData[cp][tf] = res["candles"] # In[26]: for cp in currencyPairs: for tf in timeFrames: #Reset tensorflow session and pass to gpflow tensorflow.reset_default_graph() graph = tensorflow.get_default_graph() gpflow.reset_default_session(graph=graph) #Store candle data for a specific currency pair and timeframe candles = chartData[cp][tf] #Stores the median value of candle for Y axis and #sequence between 0 and 1.10 for X data = np.zeros(shape=(N, 2)) maxVal = 0. minVal = 1000. for c in candles: median = 0.0 i = candles.index(c) closePipVal = float(candles[i]["mid"]["c"])
def train_gpsigrnn_classifier(dataset, num_hidden=128, num_levels=4, num_inducing=500, normalize_data=True, minibatch_size=50, rnn_type='lstm', use_dropout=True, max_len=500, increments=True, learn_weights=False, num_lags=None, val_split=None, test_split=None, experiment_idx=None, save_dir='./GPSigRNN/'): num_lags = num_lags or 0 rnn_type = rnn_type.lower() if rnn_type not in ['lstm', 'gru']: raise ValueError('rnn_type should be \'LSTM\' or \'GRU\'') print( '##########################################################################' ) print('Training dataset: {} | RNN = {}, num_hidden = {}, use_dropout = {}'. format(dataset, rnn_type, num_hidden, int(use_dropout))) print( '##########################################################################' ) ## load data X_train, y_train, X_val, y_val, X_test, y_test = load_dataset( dataset, val_split=val_split, test_split=test_split, max_len=max_len, normalize_data=normalize_data, add_time=True, for_model='nn') num_train, len_examples, num_features = X_train.shape num_val = X_val.shape[0] if X_val is not None else None num_test = X_test.shape[0] num_classes = np.unique(y_train).size with tf.Session(graph=tf.Graph(), config=tf.ConfigProto(gpu_options=tf.GPUOptions( allow_growth=True))) as sess: ## setup NN part of model K.set_floatx('float64') x_tens = K.placeholder(shape=(None, len_examples, num_features), dtype=gp.settings.float_type) y_tens = K.placeholder(shape=(None, 1), dtype=gp.settings.float_type) masking_layer = keras.layers.Masking(mask_value=0., input_shape=(len_examples, num_features)) recurrent_dropout = 0.05 if use_dropout else 0. dropout = 0.25 if use_dropout else 0. if rnn_type == 'lstm': rnn_layer = keras.layers.LSTM(num_hidden, recurrent_dropout=recurrent_dropout, dropout=dropout, return_sequences=True) elif rnn_type == 'gru': rnn_layer = keras.layers.GRU(num_hidden, recurrent_dropout=recurrent_dropout, dropout=dropout, return_sequences=True) # this is a hack to make reshape work with masking identity_layer = keras.layers.Lambda(lambda x: x, output_shape=lambda s: s) gp_input_dim = len_examples * num_hidden reshape_layer = keras.layers.Reshape((gp_input_dim, )) fx_tens = reshape_layer( identity_layer(rnn_layer(masking_layer(x_tens)))) nn_params = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, rnn_type) ## setup GP part of model if num_classes == 2: lik = gp.likelihoods.Bernoulli() num_latent = 1 else: lik = gp.likelihoods.MultiClass(num_classes) num_latent = num_classes # this is just temporary Z_init = np.random.randn(int(num_levels * (num_levels + 1) / 2), num_inducing, 2, num_hidden * (num_lags + 1)) feat = gpsig.inducing_variables.InducingTensors( Z_init, num_levels=num_levels, increments=increments, learn_weights=learn_weights) k = gpsig.kernels.SignatureRBF(gp_input_dim, num_levels=num_levels, num_features=num_hidden, num_lags=num_lags) if num_classes == 2: lik = gp.likelihoods.Bernoulli() num_latent = 1 else: lik = gp.likelihoods.MultiClass(num_classes) num_latent = num_classes m = gpsig.models.SVGP(fx_tens, y_tens, kern=k, feat=feat, likelihood=lik, num_latent=num_latent, minibatch_size=None, num_data=num_train) # some tensorflow magic loss = -m.likelihood_tensor fmean, fvar = m._build_predict(fx_tens) ymean, yvar = m.likelihood.predict_mean_and_var(fmean, fvar) lpd = m.likelihood.predict_density(fmean, fvar, y_tens) K.set_session(sess) sess.run(tf.variables_initializer(var_list=nn_params)) ## setup metrics def batch_predict_y(X, batch_size=None): num_iters = int(np.ceil(X.shape[0] / batch_size)) y_pred = np.zeros((X.shape[0]), dtype=np.float64) for i in range(num_iters): slice_batch = slice( i * batch_size, np.minimum((i + 1) * batch_size, X.shape[0])) X_batch = X[slice_batch] pred_batch = sess.run(ymean, feed_dict={x_tens: X_batch}) if pred_batch.shape[1] == 1: y_pred[slice_batch] = pred_batch.flatten() > 0.5 else: y_pred[slice_batch] = np.argmax(pred_batch, axis=1) return y_pred def batch_predict_density(X, y, batch_size=None): num_iters = int(np.ceil(X.shape[0] / batch_size)) y_nlpp = np.zeros((X.shape[0]), dtype=np.float64) for i in range(num_iters): slice_batch = slice( i * batch_size, np.minimum((i + 1) * batch_size, X.shape[0])) X_batch = X[slice_batch] y_batch = y[slice_batch, None] y_nlpp[slice_batch] = sess.run(lpd, feed_dict={ x_tens: X_batch, y_tens: y_batch }).flatten() return y_nlpp acc = lambda X, y: accuracy_score( y, batch_predict_y(X, batch_size=minibatch_size)) nlpp = lambda X, y: -np.mean( batch_predict_density(X, y, batch_size=minibatch_size)) val_acc = lambda: acc(X_val, y_val) val_nlpp = lambda: nlpp(X_val, y_val) test_acc = lambda: acc(X_test, y_test) test_nlpp = lambda: nlpp(X_test, y_test) ## initalize inducing points with RNN-images of random data examples fX_samples = sess.run(fx_tens, feed_dict={ x_tens: X_train[np.random.choice(num_train, size=num_inducing)] }) fX_samples = fX_samples.reshape([-1, len_examples, num_hidden]) Z_init = suggest_initial_inducing_tensors(fX_samples, num_levels, num_inducing, increments=increments, num_lags=num_lags) m.feature.Z = Z_init ## initialize lengthscales parameter of RBF kernel fX_samples = sess.run(fx_tens, feed_dict={ x_tens: X_train[np.random.choice(num_train, size=(np.minimum( 1000, num_train)), replace=False)] }) fX_samples = fX_samples.reshape([-1, len_examples, num_hidden]) l_init = suggest_initial_lengthscales(fX_samples, num_samples=1000) m.kern.lengthscales = l_init ## train_model minibatch_size = np.minimum(50, num_train) ### phase 1 - pre-train variational distribution m.kern.set_trainable(False) hist = fit_nn_with_gp_layer(X_train, y_train, m.trainable_tensors, x_tens, y_tens, loss, sess, minibatch_size=minibatch_size, max_epochs=500) ### phase 3 - train model with unfixed sigma_i m.kern.variances.set_trainable(True) var_list = nn_params + m.trainable_tensors history = fit_nn_with_gp_layer(X_train, y_train, var_list, x_tens, y_tens, loss, sess, minibatch_size=minibatch_size, max_epochs=5000, val_scores=[val_acc, val_nlpp], patience=500, lower_is_better=True, history=hist) ### restore best parameters for i, var in enumerate(var_list): sess.run(var.assign(history['best']['params'][i])) ### evaluate on validation data val_nlpp = val_nlpp() val_acc = val_acc() print('Val. nlpp: {:.4f}'.format(val_nlpp)) print('Val. accuracy: {:.4f}'.format(val_acc)) ### phase 4 - fix NN and kernel params and train on rest of data to assimilate into variational approximation #### re-merge validation data into training data if val_split is not None: X_train, y_train = np.concatenate( (X_train, X_val), axis=0), np.concatenate((y_train, y_val), axis=0) num_train = X_train.shape[0] m.num_data = num_train #### train variational distribution m.kern.set_trainable(False) history = fit_nn_with_gp_layer(X_train, y_train, m.trainable_tensors, x_tens, y_tens, loss, sess, minibatch_size=minibatch_size, max_epochs=500, history=hist) ## evaluate model test_nlpp = test_nlpp() test_acc = test_acc() test_report = classification_report( y_test, batch_predict_y(X_test, batch_size=minibatch_size)) print('Test nlpp: {:.4f}'.format(test_nlpp)) print('Test accuracy: {:.4f}'.format(test_acc)) print(test_report) ## save results to file history['results'] = {} history['results']['val_acc'] = val_acc history['results']['val_nlpp'] = val_nlpp history['results']['test_nlpp'] = test_nlpp history['results']['test_acc'] = test_acc history['results']['test_report'] = test_report if not os.path.isdir(save_dir): os.makedirs(save_dir) experiment_name = '{}_H{}_D{}'.format(dataset, num_hidden, int(use_dropout)) if experiment_idx is not None: experiment_name += '_{}'.format(experiment_idx) with open(os.path.join(save_dir, experiment_name + '.pkl'), 'wb') as f: pickle.dump(history, f) with open(os.path.join(save_dir, experiment_name + '.txt'), 'w') as f: f.write('Val. nlpp: {:.4f}\n'.format(val_nlpp)) f.write('Val. accuracy: {:.4f}\n'.format(val_acc)) f.write('Test. nlpp: {:.4f}\n'.format(test_nlpp)) f.write('Test accuracy: {:.4f}\n'.format(test_acc)) f.write('Test report:\n') f.write(test_report) # clear memory manually gp.reset_default_session() tf.reset_default_graph() K.clear_session() import gc gc.collect() return