def save(self, prepath):
makedirs_process_safe(prepath)
save_path = self.get_save_path(prepath)
np.savez(save_path,
valid=self.valid,
is_disruptive=self.is_disruptive,
signals_dict=self.signals_dict,
ttd=self.ttd)
print('...saved shot {}'.format(self.number))
def process(self, shot):
save_prepath = self.get_save_prepath()
save_path = shot.get_save_path(save_prepath)
if not os.path.exists(save_prepath):
makedirs_process_safe(save_prepath)
prepath = self.loader.conf['paths']['processed_prepath']
assert shot.valid
shot.restore(prepath)
self.loader.set_inference_mode(True) # make sure shots aren't cut
if self.loader.normalizer is not None:
self.loader.normalizer.apply(shot)
else:
print('Warning, no normalization. ',
'Training data may be poorly conditioned')
self.loader.set_inference_mode(False)
# sig, res = self.get_signal_result_from_shot(shot)
disr = 1 if shot.is_disruptive else 0
if not os.path.isfile(save_path):
X = self.get_X(shot)
np.savez(save_path, X=X) # , Y=Y, disr=disr
# print(X.shape, Y.shape)
else:
try:
dat = np.load(save_path, allow_pickle=False)
# X, Y, disr = dat["X"], dat["Y"], dat["disr"][()]
X = dat["X"]
except BaseException:
# data was there but corrupted, save it again
X = self.get_X(shot)
np.savez(save_path, X=X)
Y = self.get_Y(shot)
shot.make_light()
return X, Y, disr
def process(self,shot):
save_prepath = self.get_save_prepath()
save_path = shot.get_save_path(save_prepath)
if not os.path.exists(save_prepath):
makedirs_process_safe(save_prepath)
prepath = self.loader.conf['paths']['processed_prepath']
assert(shot.valid)
shot.restore(prepath)
self.loader.set_inference_mode(True)#make sure shots aren't cut
if self.loader.normalizer is not None:
self.loader.normalizer.apply(shot)
else:
print('Warning, no normalization. Training data may be poorly conditioned')
self.loader.set_inference_mode(False)
# sig,res = self.get_signal_result_from_shot(shot)
disr = 1 if shot.is_disruptive else 0
if not os.path.isfile(save_path):
X = self.get_X(shot)
np.savez(save_path,X=X)#,Y=Y,disr=disr
#print(X.shape,Y.shape)
else:
try:
dat = np.load(save_path)
# X,Y,disr = dat["X"],dat["Y"],dat["disr"][()]
X = dat["X"]
except: #data was there but corrupted, save it again
X = self.get_X(shot)
np.savez(save_path,X=X)
Y = self.get_Y(shot)
shot.make_light()
return X,Y,disr
def train(conf,
shot_list_train,
shot_list_validate,
loader,
shot_list_test=None):
np.random.seed(1)
print_shot_list_sizes(shot_list_train, shot_list_validate)
print('training: {} shots, {} disruptive'.format(
len(shot_list_train), shot_list_train.num_disruptive()))
print('validate: {} shots, {} disruptive'.format(
len(shot_list_validate), shot_list_validate.num_disruptive()))
num_samples = conf['model']['shallow_model']['num_samples']
feature_extractor = FeatureExtractor(loader)
shot_list_train = shot_list_train.random_sublist(debug_use_shots)
X, Y, _ = feature_extractor.load_shots(shot_list_train,
num_samples=num_samples)
Xv, Yv, _ = feature_extractor.load_shots(shot_list_validate,
num_samples=num_samples)
X = np.concatenate(X, axis=0)
Y = np.concatenate(Y, axis=0)
Xv = np.concatenate(Xv, axis=0)
Yv = np.concatenate(Yv, axis=0)
# max_samples = 100000
# num_samples = min(max_samples, len(Y))
# indices = np.random.choice(np.array(range(len(Y))), num_samples,
# replace=False)
# X = X[indices]
# Y = Y[indices]
print("fitting on {} samples, {} positive".format(len(X), np.sum(Y > 0)))
callbacks = build_callbacks(conf)
callback_metrics = conf['callbacks']['metrics']
callbacks.set_params({
'metrics': callback_metrics,
})
callbacks.on_train_begin()
callbacks.on_epoch_begin(0)
# save_prepath = feature_extractor.get_save_prepath()
model_path = (conf['paths']['model_save_path'] + model_filename
) # save_prepath + model_filename
makedirs_process_safe(conf['paths']['model_save_path'])
model_conf = conf['model']['shallow_model']
if not model_conf['skip_train'] or not os.path.isfile(model_path):
start_time = time.time()
if model_conf["scale_pos_weight"] != 1:
scale_pos_weight_dict = {
np.min(Y): 1,
np.max(Y): model_conf["scale_pos_weight"]
}
else:
scale_pos_weight_dict = None
if model_conf['type'] == "svm":
model = svm.SVC(probability=True,
C=model_conf["C"],
kernel=model_conf["kernel"],
class_weight=scale_pos_weight_dict)
elif model_conf['type'] == "random_forest":
model = RandomForestClassifier(
n_estimators=model_conf["n_estimators"],
max_depth=model_conf["max_depth"],
class_weight=scale_pos_weight_dict,
n_jobs=-1)
elif model_conf['type'] == "xgboost":
max_depth = model_conf["max_depth"]
if max_depth is None:
max_depth = 0
model = XGBClassifier(
max_depth=max_depth,
learning_rate=model_conf['learning_rate'],
n_estimators=model_conf["n_estimators"],
scale_pos_weight=model_conf["scale_pos_weight"])
elif model_conf['type'] == 'mlp':
hidden_layer_sizes = tuple(
reversed([
model_conf['final_hidden_layer_size'] * 2**x
for x in range(model_conf['num_hidden_layers'])
]))
model = MLPClassifier(
hidden_layer_sizes=hidden_layer_sizes,
learning_rate_init=model_conf['learning_rate_mlp'],
alpha=model_conf['mlp_regularization'])
else:
print("Unkown model type, exiting.")
exit(1)
model.fit(X, Y)
joblib.dump(model, model_path)
print("Fit model in {} seconds".format(time.time() - start_time))
else:
model = joblib.load(model_path)
print("model exists.")
Y_pred = model.predict(X)
print("Train")
print(classification_report(Y, Y_pred))
Y_predv = model.predict(Xv)
print("Validate")
print(classification_report(Yv, Y_predv))
if ('monitor_test' in conf['callbacks'].keys()
and conf['callbacks']['monitor_test']):
times = conf['callbacks']['monitor_times']
roc_areas, losses = make_predictions_and_evaluate_multiple_times(
conf, shot_list_validate, loader, times)
for roc, t in zip(roc_areas, times):
print('val_roc_{} = {}'.format(t, roc))
if shot_list_test is not None:
roc_areas, losses = make_predictions_and_evaluate_multiple_times(
conf, shot_list_test, loader, times)
for roc, t in zip(roc_areas, times):
print('test_roc_{} = {}'.format(t, roc))
# print(confusion_matrix(Y,Y_pred))
_, _, _, roc_area, loss = make_predictions_and_evaluate_gpu(
conf, shot_list_validate, loader)
# _, _, _, roc_area_train, loss_train = make_predictions_and_evaluate_gpu(
# conf, shot_list_train, loader)
print('Validation Loss: {:.3e}'.format(loss))
print('Validation ROC: {:.4f}'.format(roc_area))
epoch_logs = {}
epoch_logs['val_roc'] = roc_area
epoch_logs['val_loss'] = loss
# epoch_logs['train_roc'] = roc_area_train
# epoch_logs['train_loss'] = loss_train
callbacks.on_epoch_end(0, epoch_logs)
print('...done')
def train(conf, shot_list_train, shot_list_validate, loader):
np.random.seed(1)
# data_gen = ProcessGenerator(partial(
# loader.training_batch_generator_full_shot_partial_reset,shot_list
# = shot_list_train)())
data_gen = partial(loader.training_batch_generator_full_shot_partial_reset,
shot_list=shot_list_train)()
print_shot_list_sizes(shot_list_train, shot_list_validate)
loader.set_inference_mode(False)
train_model = build_torch_model(conf)
# load the latest epoch we did. Returns -1 if none exist yet
# e = specific_builder.load_model_weights(train_model)
num_epochs = conf['training']['num_epochs']
patience = conf['callbacks']['patience']
lr_decay = conf['model']['lr_decay']
# batch_size = conf['training']['batch_size']
lr = conf['model']['lr']
# clipnorm = conf['model']['clipnorm']
e = 0
# warmup_steps = conf['model']['warmup_steps']
# num_batches_minimum = conf['training']['num_batches_minimum']
# if 'adam' in conf['model']['optimizer']:
# optimizer = MPIAdam(lr=lr)
# elif conf['model']['optimizer'] == 'sgd' or conf['model']['optimizer'] ==
# 'tf_sgd':
#
# optimizer = MPISGD(lr=lr)
# elif 'momentum_sgd' in conf['model']['optimizer']:
# optimizer = MPIMomentumSGD(lr=lr)
# else:
# print("Optimizer not implemented yet")
# exit(1)
print('{} epochs left to go'.format(num_epochs - 1 - e))
if conf['callbacks']['mode'] == 'max':
best_so_far = -np.inf
cmp_fn = max
else:
best_so_far = np.inf
cmp_fn = min
optimizer = opt.Adam(train_model.parameters(), lr=lr)
scheduler = opt.lr_scheduler.ExponentialLR(optimizer, lr_decay)
train_model.train()
not_updated = 0
# total_loss = 0
# count = 0
loss_fn = nn.MSELoss(size_average=True)
model_path = get_model_path(conf)
makedirs_process_safe(os.path.dirname(model_path))
while e < num_epochs - 1:
scheduler.step()
print('\nEpoch {}/{}'.format(e, num_epochs))
(step, ave_loss, curr_loss, num_so_far,
effective_epochs) = train_epoch(train_model, data_gen, optimizer,
loss_fn)
e = effective_epochs
loader.verbose = False # True during the first iteration
# if task_index == 0:
# specific_builder.save_model_weights(train_model,int(round(e)))
torch.save(train_model.state_dict(), model_path)
_, _, _, roc_area, loss = make_predictions_and_evaluate_gpu(
conf, shot_list_validate, loader)
best_so_far = cmp_fn(roc_area, best_so_far)
# stop_training = False
print('=========Summary======== for epoch{}'.format(step))
print('Training Loss numpy: {:.3e}'.format(ave_loss))
print('Validation Loss: {:.3e}'.format(loss))
print('Validation ROC: {:.4f}'.format(roc_area))
# only save model weights if the quantity we are tracking is improving
if best_so_far != roc_area:
print("No improvement, still saving model")
not_updated += 1
else:
print("Saving model")
# specific_builder.delete_model_weights(train_model,int(round(e)))
if not_updated > patience:
print("Stopping training due to early stopping")
break
def ensure_save_directory(self):
prepath = self.conf['paths']['model_save_path']
makedirs_process_safe(prepath)
def save(self,prepath):
makedirs_process_safe(prepath)
save_path = self.get_save_path(prepath)
np.savez(save_path,valid=self.valid,is_disruptive=self.is_disruptive,
signals_dict=self.signals_dict,ttd=self.ttd)
print('...saved shot {}'.format(self.number))
def ensure_save_directory(self):
prepath = self.conf['paths']['model_save_path']
makedirs_process_safe(prepath)
def train(conf,shot_list_train,shot_list_validate,loader):
np.random.seed(1)
print('validate: {} shots, {} disruptive'.format(len(shot_list_validate),shot_list_validate.num_disruptive()))
print('training: {} shots, {} disruptive'.format(len(shot_list_train),shot_list_train.num_disruptive()))
num_samples = conf['model']['shallow_model']['num_samples']
feature_extractor = FeatureExtractor(loader)
shot_list_train = shot_list_train.random_sublist(debug_use_shots)
X,Y,_ = feature_extractor.load_shots(shot_list_train,num_samples = num_samples)
Xv,Yv,_ = feature_extractor.load_shots(shot_list_validate,num_samples = num_samples)
X = np.concatenate(X,axis=0)
Y = np.concatenate(Y,axis=0)
Xv = np.concatenate(Xv,axis=0)
Yv = np.concatenate(Yv,axis=0)
#max_samples = 100000
#num_samples = min(max_samples,len(Y))
#indices = np.random.choice(np.array(range(len(Y))),num_samples,replace=False)
#X = X[indices]
#Y = Y[indices]
print("fitting on {} samples, {} positive".format(len(X),np.sum(Y > 0)))
callbacks = build_callbacks(conf)
callback_metrics = conf['callbacks']['metrics']
callbacks.set_params({
'metrics': callback_metrics,
})
callbacks.on_train_begin()
callbacks.on_epoch_begin(0)
save_prepath = feature_extractor.get_save_prepath()
model_path = conf['paths']['model_save_path'] + model_filename #save_prepath + model_filename
makedirs_process_safe(conf['paths']['model_save_path'])
model_conf = conf['model']['shallow_model']
if not model_conf['skip_train'] or not os.path.isfile(model_path):
start_time = time.time()
if model_conf["scale_pos_weight"] != 1:
scale_pos_weight_dict = {np.min(Y) : 1, np.max(Y):model_conf["scale_pos_weight"]}
else:
scale_pos_weight_dict = None
if model_conf['type'] == "svm":
model = svm.SVC(probability=True,
C=model_conf["C"],
kernel=model_conf["kernel"],
class_weight=scale_pos_weight_dict)
elif model_conf['type'] == "random_forest":
model = RandomForestClassifier(n_estimators=model_conf["n_estimators"],
max_depth=model_conf["max_depth"],
class_weight=scale_pos_weight_dict,
n_jobs=-1)
elif model_conf['type'] == "xgboost":
max_depth = model_conf["max_depth"]
if max_depth == None:
max_depth = 0
model = XGBClassifier(max_depth=max_depth,
learning_rate=model_conf['learning_rate'],
n_estimators=model_conf["n_estimators"],
scale_pos_weight=model_conf["scale_pos_weight"])
elif model_conf['type'] == 'mlp':
hidden_layer_sizes = tuple(reversed([model_conf['final_hidden_layer_size']*2**x for x in range(model_conf['num_hidden_layers'])]))
model = MLPClassifier(hidden_layer_sizes = hidden_layer_sizes,
learning_rate_init = model_conf['learning_rate_mlp'],
alpha = model_conf['mlp_regularization'])
else:
print("Unkown model type, exiting.")
exit(1)
model.fit(X,Y)
joblib.dump(model,model_path)
print("Fit model in {} seconds".format(time.time()-start_time))
else:
model = joblib.load(model_path)
print("model exists.")
Y_pred = model.predict(X)
print("Train")
print(classification_report(Y,Y_pred))
Y_predv = model.predict(Xv)
print("Validate")
print(classification_report(Yv,Y_predv))
#print(confusion_matrix(Y,Y_pred))
_,_,_,roc_area,loss = make_predictions_and_evaluate_gpu(conf,shot_list_validate,loader)
# _,_,_,roc_area_train,loss_train = make_predictions_and_evaluate_gpu(conf,shot_list_train,loader)
print('Validation Loss: {:.3e}'.format(loss))
print('Validation ROC: {:.4f}'.format(roc_area))
epoch_logs = {}
epoch_logs['val_roc'] = roc_area
epoch_logs['val_loss'] = loss
# epoch_logs['train_roc'] = roc_area_train
# epoch_logs['train_loss'] = loss_train
callbacks.on_epoch_end(0, epoch_logs)
print('...done')
