def test_client_test_mode_send_metrics_does_nothing(): client = sherpa.Client(test_mode=True) trial = client.get_trial() assert trial.id == 1 assert trial.parameters == {} client.send_metrics(trial=trial, iteration=1, objective=0.1)
def test_client_test_mode(): client = sherpa.Client(test_mode=True) trial = client.get_trial() assert trial.id == 1 assert trial.parameters == {} client.send_metrics(trial=trial, iteration=1, objective=0.1) callback = client.keras_send_metrics(trial=trial, objective_name='val_acc', context_names=['val_loss', 'loss', 'acc'])
def test_database(test_dir): test_trial = get_test_trial() testlogger.debug(test_dir) db_port = sherpa.core._port_finder(27000, 28000) with sherpa.database._Database(test_dir, port=db_port) as db: time.sleep(2) testlogger.debug("Enqueuing...") db.enqueue_trial(test_trial) testlogger.debug("Starting Client...") client = sherpa.Client(port=db_port, connectTimeoutMS=100, serverSelectionTimeoutMS=1000) testlogger.debug("Getting Trial...") os.environ['SHERPA_TRIAL_ID'] = '1' t = client.get_trial() assert t.id == 1 assert t.parameters == {'a': 1, 'b': 2} testlogger.debug("Sending Metrics...") client.send_metrics(trial=t, iteration=1, objective=0.1, context={'other_metric': 0.2}) new_results = db.get_new_results() testlogger.debug(new_results) assert new_results == [{ 'context': { 'other_metric': 0.2 }, 'iteration': 1, 'objective': 0.1, 'parameters': { 'a': 1, 'b': 2 }, 'trial_id': 1 }] # test that Sherpa raises correct error if MongoDB exits db2 = sherpa.database._Database(test_dir, port=db_port) with pytest.raises(OSError): db2.start()
help='Store visualizations of weights') parser.add_argument('--gif_weights', default=False, action='store_true', help='Store gif visualizations of weights') parser.add_argument('--env', type=str, default='MNIST', choices=['MNIST', 'SMNIST', 'LMNIST', 'GYM'], help='Type of task') parser.add_argument('--record', type=str, default='AND', choices=['AND', 'OR']) args = vars(parser.parse_args()) ########################### import sherpa client = sherpa.Client(test_mode=args['notsherpa']) trial = client.get_trial() ########################### if args['notsherpa']: from config import settings args.update(settings) else: args = trial.parameters # pretty print settings pp = pprint.PrettyPrinter(indent=4) pp.pprint(args) env = EnvManager(args) monitor = AgentMonitor(args)
test['prob'] = test_probabilities[:, 1] df = train.append(test, ignore_index=True) print('SAVING TO:', 'Models/%d/%05d.csv' % (FLAGS.sherpa_trial, fold + 1), df.shape) df.to_csv('Models/%d/%05d.csv' % (FLAGS.sherpa_trial, fold + 1)) plot_confusion_matrix(train_cm / 10., FLAGS, prefix='train_') plot_confusion_matrix(test_cm / 10., FLAGS, prefix='test_') else: # USING SHERPA if not FLAGS.notsherpa: client = sherpa.Client() trial = client.get_trial() # UPDATE DEFAULTS WITH TRIAL args.update(trial.parameters) result_file_name = 'Results/sherpa_results_new.txt' pp.pprint(args) fold_results = { 'auc': [], 'loss': [], 'acc': [], 'val_auc': [], 'val_loss': [],
CONDA_DIR = '/work/00157/walling/conda/envs/CbrainCustomLayer/' DATA_DIR = '/work/00157/walling/projects/cloud_emulator/walling-CBRAIN-CAM/notebooks/tbeucler_devlog/sherpa/data/' SHERPA_TEMP_DIR = '/work/00157/walling/projects/cloud_emulator/walling-CBRAIN-CAM/notebooks/tbeucler_devlog/sherpa/sherpa-temp-study1/' EPOCHS = 10 #TRAINDIR = DATA_DIR os.chdir(DEV_DIR) sys.path.insert(1, CONDA_DIR + "lib/python3.7/site-packages") #work around for h5py sys.path.insert(1, DEV_DIR) sys.path.insert(1, DEV_DIR + "cbrain") sys.path.insert(1, DEV_DIR + "notebooks/tbeucler_devlog") import sherpa client = sherpa.Client(host='127.0.0.1') #, port='37001') trial = client.get_trial() BATCH_SIZE = trial.parameters['batch_size'] NUM_LAYERS = trial.parameters['num_layers'] NUM_UNITS = trial.parameters['num_units'] LEARNING_RATE = trial.parameters['learning_rate'] DROPOUT_RATE = trial.parameters['dropout_rate'] LEAKY_RELU_ALPHA = trial.parameters['leaky_relu_alpha'] BATCH_NORM = trial.parameters['batch_norm'] print('Batch Size = ' + str(BATCH_SIZE)) print('Num Layers = ' + str(NUM_LAYERS)) print('Num Units = ' + str(NUM_UNITS)) print('Learning Rate = ' + str(LEARNING_RATE)) print('Dropout Rate = ' + str(DROPOUT_RATE))
16 seconds per epoch on a GRID K520 GPU. ''' from __future__ import print_function import os import argparse parser = argparse.ArgumentParser() parser.add_argument('--notsherpa', help='Do not import Sherpa', action='store_true', default=False) parser.add_argument('--gpu', type=str, default='') args, unknown = parser.parse_known_args() import sherpa client = sherpa.Client(test_mode=args.notsherpa) trial = client.get_trial() gpu = os.environ.get("SHERPA_RESOURCE", '') os.environ['CUDA_VISIBLE_DEVICES'] = str(gpu or args.gpu) os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' import tensorflow as tf CONFIG = tf.ConfigProto(device_count={'GPU': 1}, log_device_placement=False, allow_soft_placement=False) CONFIG.gpu_options.allow_growth = True # Prevents tf from grabbing all gpu memory. sess = tf.Session(config=CONFIG) from keras import backend as K K.set_session(sess)