def make_single_prediction(shot, specific_builder, loader, model_save_path):
loader.set_inference_mode(True)
model = specific_builder.build_model(True)
model.compile(optimizer=optimizer_class(),
loss=conf['data']['target'].loss)
model.load_weights(model_save_path)
model.reset_states()
X, y = loader.load_as_X_y(shot, prediction_mode=True)
assert (X.shape[0] == y.shape[0])
y_p = model.predict(X,
batch_size=Loader.get_batch_size(
conf['training']['batch_size'],
prediction_mode=True),
verbose=0)
answer_dims = y_p.shape[-1]
if conf['model']['return_sequences']:
shot_length = y_p.shape[0] * y_p.shape[1]
else:
shot_length = y_p.shape[0]
y_p = np.reshape(y_p, (shot_length, answer_dims))
y = np.reshape(y, (shot_length, answer_dims))
is_disruptive = shot.is_disruptive_shot()
model.reset_states()
loader.set_inference_mode(False)
return y_p, y, is_disruptive
def make_single_prediction(shot,specific_builder,loader,model_save_path):
loader.set_inference_mode(True)
model = specific_builder.build_model(True)
model.compile(optimizer=optimizer_class(),loss=conf['data']['target'].loss)
model.load_weights(model_save_path)
model.reset_states()
X,y = loader.load_as_X_y(shot,prediction_mode=True)
assert(X.shape[0] == y.shape[0])
y_p = model.predict(X,batch_size=Loader.get_batch_size(conf['training']['batch_size'],prediction_mode=True),verbose=0)
answer_dims = y_p.shape[-1]
if conf['model']['return_sequences']:
shot_length = y_p.shape[0]*y_p.shape[1]
else:
shot_length = y_p.shape[0]
y_p = np.reshape(y_p,(shot_length,answer_dims))
y = np.reshape(y,(shot_length,answer_dims))
is_disruptive = shot.is_disruptive_shot()
model.reset_states()
loader.set_inference_mode(False)
return y_p,y,is_disruptive
# if not, only master MPI rank spawns thread pool to perform preprocessing
(shot_list_train, shot_list_validate,
shot_list_test) = guarantee_preprocessed(conf)
# similarly, train normalizer (if necessary) w/ master MPI rank only
normalizer.train() # verbose=False only suppresses if purely loading
g.comm.Barrier()
g.print_unique("begin preprocessor+normalization (all MPI ranks)...")
# second call has ALL MPI ranks load preprocessed shots from .npz files
(shot_list_train, shot_list_validate,
shot_list_test) = guarantee_preprocessed(conf, verbose=True)
# second call to normalizer training
normalizer.conf['data']['recompute_normalization'] = False
normalizer.train(verbose=True)
# KGF: may want to set it back...
# normalizer.conf['data']['recompute_normalization'] = conf['data']['recompute_normalization'] # noqa
loader = Loader(conf, normalizer)
g.print_unique("...done")
# TODO(KGF): both preprocess.py and normalize.py are littered with print()
# calls that should probably be replaced with print_unique() when they are not
# purely loading previously-computed quantities from file
# (or we can continue to ensure that they are only ever executed by 1 rank)
#####################################################
# TRAINING #
#####################################################
# Prevent Keras TF backend deprecation messages from mpi_train() from
# appearing jumbled with stdout, stderr msgs from above steps
g.comm.Barrier()
g.flush_all_inorder()
return t_range,get_importance_measure_given_y_prime(y_prime,metric),y_prime[-1]
def difference_metric(y_prime,y_prime_orig):
idx = np.argmax(y_prime_orig)
return (np.max(y_prime_orig) - y_prime[idx])/(np.max(y_prime_orig) - np.min(y_prime_orig))
def get_importance_measure_given_y_prime(y_prime,metric):
differences = [metric(y_prime[i],y_prime[-1]) for i in range(len(y_prime))]
return 1.0-np.array(differences)#/np.max(differences)
print("normalization",end='')
normalizer = Normalizer(conf)
normalizer.train()
normalizer = ByShotAugmentator(normalizer)
loader = Loader(conf,normalizer)
print("...done")
# if not only_predict:
# mpi_train(conf,shot_list_train,shot_list_validate,loader)
#load last model for testing
loader.set_inference_mode(True)
use_signals = copy.copy(conf['paths']['use_signals'])
use_signals.append(None)
for shot in shot_list_test:
shot.augmentation_fn = None# partial(hide_signal_data,t = 0,sigs_to_hide = sigs_to_hide)
def keras_fmin_fnct(self, space):
from plasma.models import builder
specific_builder = builder.ModelBuilder(self.conf)
train_model = specific_builder.hyper_build_model(space, False)
train_model.compile(optimizer=optimizer_class(),
loss=conf['data']['target'].loss)
np.random.seed(1)
validation_losses = []
validation_roc = []
training_losses = []
shot_list_train, shot_list_validate = self.shot_list.split_direct(
1.0 - conf['training']['validation_frac'], do_shuffle=True)
from keras.utils.generic_utils import Progbar
from keras import backend as K
num_epochs = self.conf['training']['num_epochs']
num_at_once = self.conf['training']['num_shots_at_once']
lr_decay = self.conf['model']['lr_decay']
resulting_dict = {'loss': None, 'status': STATUS_OK, 'model': None}
e = -1
#print("Current num_epochs {}".format(e))
while e < num_epochs - 1:
e += 1
pbar = Progbar(len(shot_list_train))
shot_list_train.shuffle()
shot_sublists = shot_list_train.sublists(num_at_once)[:1]
training_losses_tmp = []
K.set_value(train_model.optimizer.lr, lr * lr_decay**(e))
for (i, shot_sublist) in enumerate(shot_sublists):
X_list, y_list = self.loader.load_as_X_y_list(shot_sublist)
for j, (X, y) in enumerate(zip(X_list, y_list)):
history = builder.LossHistory()
train_model.fit(X,
y,
batch_size=Loader.get_batch_size(
self.conf['training']['batch_size'],
prediction_mode=False),
epochs=1,
shuffle=False,
verbose=0,
validation_split=0.0,
callbacks=[history])
train_model.reset_states()
train_loss = np.mean(history.losses)
training_losses_tmp.append(train_loss)
pbar.add(1.0 * len(shot_sublist) / len(X_list),
values=[("train loss", train_loss)])
self.loader.verbose = False
sys.stdout.flush()
training_losses.append(np.mean(training_losses_tmp))
specific_builder.save_model_weights(train_model, e)
_, _, _, roc_area, loss = make_predictions_and_evaluate_gpu(
self.conf, shot_list_validate, self.loader)
print("Epoch: {}, loss: {}, validation_losses_size: {}".format(
e, loss, len(validation_losses)))
validation_losses.append(loss)
validation_roc.append(roc_area)
resulting_dict['loss'] = loss
resulting_dict['model'] = train_model
#print("Results {}, before {}".format(resulting_dict,id(resulting_dict)))
#print("Results {}, after {}".format(resulting_dict,id(resulting_dict)))
return resulting_dict
#####################################################
####################Normalization####################
#####################################################
if task_index == 0: #make sure preprocessing has been run, and is saved as a file
shot_list_train, shot_list_validate, shot_list_test = guarantee_preprocessed(
conf)
comm.Barrier()
shot_list_train, shot_list_validate, shot_list_test = guarantee_preprocessed(
conf)
print("normalization", end='')
raw_normalizer = Normalizer(conf)
raw_normalizer.train()
is_inference = False
normalizer = Augmentator(raw_normalizer, is_inference, conf)
loader = Loader(conf, normalizer)
print("...done")
if not only_predict:
mpi_train(conf, shot_list_train, shot_list_validate, loader)
#load last model for testing
print('saving results')
y_prime = []
y_gold = []
disruptive = []
# y_prime_train,y_gold_train,disruptive_train = make_predictions(conf,shot_list_train,loader)
# y_prime_test,y_gold_test,disruptive_test = make_predictions(conf,shot_list_test,loader)
normalizer.set_inference(True)
np.random.seed(0)
random.seed(0)
#####################################################
####################PREPROCESSING####################
#####################################################
shot_list_train, shot_list_validate, shot_list_test = guarantee_preprocessed(
conf)
#####################################################
####################Normalization####################
#####################################################
print("normalization", end='')
nn = Normalizer(conf)
nn.train()
loader = Loader(conf, nn)
print("...done")
print('Training on {} shots, testing on {} shots'.format(
len(shot_list_train), len(shot_list_test)))
#####################################################
######################TRAINING#######################
#####################################################
#train(conf,shot_list_train,loader)
p = old_mp.Process(target=train,
args=(conf, shot_list_train, shot_list_validate, loader))
p.start()
p.join()
#####################################################
####################PREDICTING#######################
elif conf['data']['normalizer'] == 'meanvar':
from plasma.preprocessor.normalize import MeanVarNormalizer as Normalizer
elif conf['data']['normalizer'] == 'var':
from plasma.preprocessor.normalize import VarNormalizer as Normalizer #performs !much better than minmaxnormalizer
elif conf['data']['normalizer'] == 'averagevar':
from plasma.preprocessor.normalize import AveragingVarNormalizer as Normalizer #performs !much better than minmaxnormalizer
else:
print('unkown normalizer. exiting')
exit(1)
np.random.seed(1)
print("normalization", end='')
nn = Normalizer(conf)
nn.train()
loader = Loader(conf, nn)
shot_list_train, shot_list_validate, shot_list_test = loader.load_shotlists(
conf)
print("...done")
print('Training on {} shots, testing on {} shots'.format(
len(shot_list_train), len(shot_list_test)))
from plasma.models import runner
specific_runner = runner.HyperRunner(conf, loader, shot_list_train)
best_run, best_model = specific_runner.frnn_minimize(algo=tpe.suggest,
max_evals=2,
trials=Trials())
print(best_run)
print(best_model)
if conf['data']['normalizer'] == 'minmax':
from plasma.preprocessor.normalize import MinMaxNormalizer as Normalizer
elif conf['data']['normalizer'] == 'meanvar':
from plasma.preprocessor.normalize import MeanVarNormalizer as Normalizer
elif conf['data']['normalizer'] == 'var':
from plasma.preprocessor.normalize import VarNormalizer as Normalizer #performs !much better than minmaxnormalizer
elif conf['data']['normalizer'] == 'averagevar':
from plasma.preprocessor.normalize import AveragingVarNormalizer as Normalizer #performs !much better than minmaxnormalizer
else:
print('unkown normalizer. exiting')
exit(1)
np.random.seed(1)
print("normalization",end='')
nn = Normalizer(conf)
nn.train()
loader = Loader(conf,nn)
shot_list_train,shot_list_validate,shot_list_test = loader.load_shotlists(conf)
print("...done")
print('Training on {} shots, testing on {} shots'.format(len(shot_list_train),len(shot_list_test)))
from plasma.models import runner
specific_runner = runner.HyperRunner(conf,loader,shot_list_train)
best_run, best_model = specific_runner.frnn_minimize(algo=tpe.suggest,max_evals=2,trials=Trials())
print (best_run)
print (best_model)
def keras_fmin_fnct(self,space):
from plasma.models import builder
specific_builder = builder.ModelBuilder(self.conf)
train_model = specific_builder.hyper_build_model(space,False)
train_model.compile(optimizer=optimizer_class(),loss=conf['data']['target'].loss)
np.random.seed(1)
validation_losses = []
validation_roc = []
training_losses = []
shot_list_train,shot_list_validate = self.shot_list.split_direct(1.0-conf['training']['validation_frac'],do_shuffle=True)
from keras.utils.generic_utils import Progbar
from keras import backend as K
num_epochs = self.conf['training']['num_epochs']
num_at_once = self.conf['training']['num_shots_at_once']
lr_decay = self.conf['model']['lr_decay']
resulting_dict = {'loss':None,'status':STATUS_OK,'model':None}
e = -1
#print("Current num_epochs {}".format(e))
while e < num_epochs-1:
e += 1
pbar = Progbar(len(shot_list_train))
shot_list_train.shuffle()
shot_sublists = shot_list_train.sublists(num_at_once)[:1]
training_losses_tmp = []
K.set_value(train_model.optimizer.lr, lr*lr_decay**(e))
for (i,shot_sublist) in enumerate(shot_sublists):
X_list,y_list = self.loader.load_as_X_y_list(shot_sublist)
for j,(X,y) in enumerate(zip(X_list,y_list)):
history = builder.LossHistory()
train_model.fit(X,y,
batch_size=Loader.get_batch_size(self.conf['training']['batch_size'],prediction_mode=False),
epochs=1,shuffle=False,verbose=0,
validation_split=0.0,callbacks=[history])
train_model.reset_states()
train_loss = np.mean(history.losses)
training_losses_tmp.append(train_loss)
pbar.add(1.0*len(shot_sublist)/len(X_list), values=[("train loss", train_loss)])
self.loader.verbose=False
sys.stdout.flush()
training_losses.append(np.mean(training_losses_tmp))
specific_builder.save_model_weights(train_model,e)
_,_,_,roc_area,loss = make_predictions_and_evaluate_gpu(self.conf,shot_list_validate,self.loader)
print("Epoch: {}, loss: {}, validation_losses_size: {}".format(e,loss,len(validation_losses)))
validation_losses.append(loss)
validation_roc.append(roc_area)
resulting_dict['loss'] = loss
resulting_dict['model'] = train_model
#print("Results {}, before {}".format(resulting_dict,id(resulting_dict)))
#print("Results {}, after {}".format(resulting_dict,id(resulting_dict)))
return resulting_dict