y_prime,y_gold,disruptive = mpi_make_predictions(conf,shot_list_tmp,loader,custom_path)
shot_list_tmp.make_light()
return t_range,get_importance_measure_given_y_prime(y_prime,metric),y_prime[-1]
def difference_metric(y_prime,y_prime_orig):
idx = np.argmax(y_prime_orig)
return (np.max(y_prime_orig) - y_prime[idx])/(np.max(y_prime_orig) - np.min(y_prime_orig))
def get_importance_measure_given_y_prime(y_prime,metric):
differences = [metric(y_prime[i],y_prime[-1]) for i in range(len(y_prime))]
return 1.0-np.array(differences)#/np.max(differences)
print("normalization",end='')
normalizer = Normalizer(conf)
normalizer.train()
normalizer = ByShotAugmentator(normalizer)
loader = Loader(conf,normalizer)
print("...done")
# if not only_predict:
# mpi_train(conf,shot_list_train,shot_list_validate,loader)
#load last model for testing
loader.set_inference_mode(True)
use_signals = copy.copy(conf['paths']['use_signals'])
use_signals.append(None)
for shot in shot_list_test:
custom_path = sys.argv[2]
print("predicting using path {}".format(custom_path))
#####################################################
####################Normalization####################
#####################################################
if task_index == 0: #make sure preprocessing has been run, and is saved as a file
shot_list_train, shot_list_validate, shot_list_test = guarantee_preprocessed(
conf)
comm.Barrier()
shot_list_train, shot_list_validate, shot_list_test = guarantee_preprocessed(
conf)
print("normalization", end='')
normalizer = Normalizer(conf)
normalizer.train()
loader = Loader(conf, normalizer)
print("...done")
#ensure training has a separate random seed for every worker
np.random.seed(task_index)
random.seed(task_index)
if not only_predict:
mpi_train(conf, shot_list_train, shot_list_validate, loader)
#load last model for testing
loader.set_inference_mode(True)
print('saving results')
y_prime = []
y_gold = []
disruptive = []
custom_path = sys.argv[1]
print("predicting using path {}".format(custom_path))
#####################################################
####################Normalization####################
#####################################################
if task_index == 0: #make sure preprocessing has been run, and is saved as a file
shot_list_train, shot_list_validate, shot_list_test = guarantee_preprocessed(
conf)
comm.Barrier()
shot_list_train, shot_list_validate, shot_list_test = guarantee_preprocessed(
conf)
print("normalization", end='')
raw_normalizer = Normalizer(conf)
raw_normalizer.train()
is_inference = False
normalizer = Augmentator(raw_normalizer, is_inference, conf)
loader = Loader(conf, normalizer)
print("...done")
if not only_predict:
mpi_train(conf, shot_list_train, shot_list_validate, loader)
#load last model for testing
print('saving results')
y_prime = []
y_gold = []
disruptive = []
# y_prime_train,y_gold_train,disruptive_train = make_predictions(conf,shot_list_train,loader)
np.random.seed(0)
random.seed(0)
#####################################################
####################PREPROCESSING####################
#####################################################
shot_list_train, shot_list_validate, shot_list_test = guarantee_preprocessed(
conf)
#####################################################
####################Normalization####################
#####################################################
print("normalization", end='')
nn = Normalizer(conf)
nn.train()
loader = Loader(conf, nn)
print("...done")
print('Training on {} shots, testing on {} shots'.format(
len(shot_list_train), len(shot_list_test)))
#####################################################
######################TRAINING#######################
#####################################################
#train(conf,shot_list_train,loader)
p = old_mp.Process(target=train,
args=(conf, shot_list_train, shot_list_validate, loader))
p.start()
p.join()
#####################################################
if conf['data']['normalizer'] == 'minmax':
from plasma.preprocessor.normalize import MinMaxNormalizer as Normalizer
elif conf['data']['normalizer'] == 'meanvar':
from plasma.preprocessor.normalize import MeanVarNormalizer as Normalizer
elif conf['data']['normalizer'] == 'var':
from plasma.preprocessor.normalize import VarNormalizer as Normalizer #performs !much better than minmaxnormalizer
elif conf['data']['normalizer'] == 'averagevar':
from plasma.preprocessor.normalize import AveragingVarNormalizer as Normalizer #performs !much better than minmaxnormalizer
else:
print('unkown normalizer. exiting')
exit(1)
np.random.seed(1)
print("normalization",end='')
nn = Normalizer(conf)
nn.train()
loader = Loader(conf,nn)
shot_list_train,shot_list_validate,shot_list_test = loader.load_shotlists(conf)
print("...done")
print('Training on {} shots, testing on {} shots'.format(len(shot_list_train),len(shot_list_test)))
from plasma.models import runner
specific_runner = runner.HyperRunner(conf,loader,shot_list_train)
best_run, best_model = specific_runner.frnn_minimize(algo=tpe.suggest,max_evals=2,trials=Trials())
print (best_run)
print (best_model)
custom_path = sys.argv[1]
print("predicting using path {}".format(custom_path))
#####################################################
####################Normalization####################
#####################################################
if task_index == 0: #make sure preprocessing has been run, and is saved as a file
shot_list_train,shot_list_validate,shot_list_test = guarantee_preprocessed(conf)
comm.Barrier()
shot_list_train,shot_list_validate,shot_list_test = guarantee_preprocessed(conf)
print("normalization",end='')
raw_normalizer = Normalizer(conf)
raw_normalizer.train()
is_inference= False
normalizer = Augmentator(raw_normalizer,is_inference,conf)
loader = Loader(conf,normalizer)
print("...done")
if not only_predict:
mpi_train(conf,shot_list_train,shot_list_validate,loader)
#load last model for testing
print('saving results')
y_prime = []
y_gold = []
disruptive= []
# y_prime_train,y_gold_train,disruptive_train = make_predictions(conf,shot_list_train,loader)