def test_client_test_mode_send_metrics_does_nothing():
client = sherpa.Client(test_mode=True)
trial = client.get_trial()
assert trial.id == 1
assert trial.parameters == {}
client.send_metrics(trial=trial, iteration=1, objective=0.1)
def test_client_test_mode():
client = sherpa.Client(test_mode=True)
trial = client.get_trial()
assert trial.id == 1
assert trial.parameters == {}
client.send_metrics(trial=trial, iteration=1, objective=0.1)
callback = client.keras_send_metrics(trial=trial, objective_name='val_acc', context_names=['val_loss', 'loss', 'acc'])
def test_database(test_dir):
test_trial = get_test_trial()
testlogger.debug(test_dir)
db_port = sherpa.core._port_finder(27000, 28000)
with sherpa.database._Database(test_dir, port=db_port) as db:
time.sleep(2)
testlogger.debug("Enqueuing...")
db.enqueue_trial(test_trial)
testlogger.debug("Starting Client...")
client = sherpa.Client(port=db_port,
connectTimeoutMS=100,
serverSelectionTimeoutMS=1000)
testlogger.debug("Getting Trial...")
os.environ['SHERPA_TRIAL_ID'] = '1'
t = client.get_trial()
assert t.id == 1
assert t.parameters == {'a': 1, 'b': 2}
testlogger.debug("Sending Metrics...")
client.send_metrics(trial=t,
iteration=1,
objective=0.1,
context={'other_metric': 0.2})
new_results = db.get_new_results()
testlogger.debug(new_results)
assert new_results == [{
'context': {
'other_metric': 0.2
},
'iteration': 1,
'objective': 0.1,
'parameters': {
'a': 1,
'b': 2
},
'trial_id': 1
}]
# test that Sherpa raises correct error if MongoDB exits
db2 = sherpa.database._Database(test_dir, port=db_port)
with pytest.raises(OSError):
db2.start()
help='Store visualizations of weights')
parser.add_argument('--gif_weights',
default=False,
action='store_true',
help='Store gif visualizations of weights')
parser.add_argument('--env',
type=str,
default='MNIST',
choices=['MNIST', 'SMNIST', 'LMNIST', 'GYM'],
help='Type of task')
parser.add_argument('--record', type=str, default='AND', choices=['AND', 'OR'])
args = vars(parser.parse_args())
###########################
import sherpa
client = sherpa.Client(test_mode=args['notsherpa'])
trial = client.get_trial()
###########################
if args['notsherpa']:
from config import settings
args.update(settings)
else:
args = trial.parameters
# pretty print settings
pp = pprint.PrettyPrinter(indent=4)
pp.pprint(args)
env = EnvManager(args)
monitor = AgentMonitor(args)
test['prob'] = test_probabilities[:, 1]
df = train.append(test, ignore_index=True)
print('SAVING TO:',
'Models/%d/%05d.csv' % (FLAGS.sherpa_trial, fold + 1), df.shape)
df.to_csv('Models/%d/%05d.csv' % (FLAGS.sherpa_trial, fold + 1))
plot_confusion_matrix(train_cm / 10., FLAGS, prefix='train_')
plot_confusion_matrix(test_cm / 10., FLAGS, prefix='test_')
else:
# USING SHERPA
if not FLAGS.notsherpa:
client = sherpa.Client()
trial = client.get_trial()
# UPDATE DEFAULTS WITH TRIAL
args.update(trial.parameters)
result_file_name = 'Results/sherpa_results_new.txt'
pp.pprint(args)
fold_results = {
'auc': [],
'loss': [],
'acc': [],
'val_auc': [],
'val_loss': [],
CONDA_DIR = '/work/00157/walling/conda/envs/CbrainCustomLayer/'
DATA_DIR = '/work/00157/walling/projects/cloud_emulator/walling-CBRAIN-CAM/notebooks/tbeucler_devlog/sherpa/data/'
SHERPA_TEMP_DIR = '/work/00157/walling/projects/cloud_emulator/walling-CBRAIN-CAM/notebooks/tbeucler_devlog/sherpa/sherpa-temp-study1/'
EPOCHS = 10
#TRAINDIR = DATA_DIR
os.chdir(DEV_DIR)
sys.path.insert(1, CONDA_DIR +
"lib/python3.7/site-packages") #work around for h5py
sys.path.insert(1, DEV_DIR)
sys.path.insert(1, DEV_DIR + "cbrain")
sys.path.insert(1, DEV_DIR + "notebooks/tbeucler_devlog")
import sherpa
client = sherpa.Client(host='127.0.0.1') #, port='37001')
trial = client.get_trial()
BATCH_SIZE = trial.parameters['batch_size']
NUM_LAYERS = trial.parameters['num_layers']
NUM_UNITS = trial.parameters['num_units']
LEARNING_RATE = trial.parameters['learning_rate']
DROPOUT_RATE = trial.parameters['dropout_rate']
LEAKY_RELU_ALPHA = trial.parameters['leaky_relu_alpha']
BATCH_NORM = trial.parameters['batch_norm']
print('Batch Size = ' + str(BATCH_SIZE))
print('Num Layers = ' + str(NUM_LAYERS))
print('Num Units = ' + str(NUM_UNITS))
print('Learning Rate = ' + str(LEARNING_RATE))
print('Dropout Rate = ' + str(DROPOUT_RATE))
16 seconds per epoch on a GRID K520 GPU.
'''
from __future__ import print_function
import os
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('--notsherpa',
help='Do not import Sherpa',
action='store_true',
default=False)
parser.add_argument('--gpu', type=str, default='')
args, unknown = parser.parse_known_args()
import sherpa
client = sherpa.Client(test_mode=args.notsherpa)
trial = client.get_trial()
gpu = os.environ.get("SHERPA_RESOURCE", '')
os.environ['CUDA_VISIBLE_DEVICES'] = str(gpu or args.gpu)
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
import tensorflow as tf
CONFIG = tf.ConfigProto(device_count={'GPU': 1},
log_device_placement=False,
allow_soft_placement=False)
CONFIG.gpu_options.allow_growth = True # Prevents tf from grabbing all gpu memory.
sess = tf.Session(config=CONFIG)
from keras import backend as K
K.set_session(sess)