def run(ex, train_filename, test_filename, n_folds, i_test_fold,
valid_set_fraction, use_validation_set, low_cut_hz, model_name,
optimizer_name, init_lr, scheduler_name, use_norm_constraint, restarts,
weight_decay, schedule_weight_decay, max_epochs, max_increase_epochs,
np_th_seed, debug):
kwargs = locals()
kwargs.pop('ex')
import sys
logging.basicConfig(format='%(asctime)s %(levelname)s : %(message)s',
level=logging.DEBUG,
stream=sys.stdout)
start_time = time.time()
ex.info['finished'] = False
# check that gpu is available -> should lead to crash if gpu not there
confirm_gpu_availability()
exp = run_4_sec_exp(**kwargs)
end_time = time.time()
last_row = exp.epochs_df.iloc[-1]
end_time = time.time()
run_time = end_time - start_time
ex.info['finished'] = True
for key, val in last_row.iteritems():
ex.info[key] = float(val)
ex.info['runtime'] = run_time
save_pkl_artifact(ex, exp.epochs_df, 'epochs_df.pkl')
save_pkl_artifact(ex, exp.before_stop_df, 'before_stop_df.pkl')
def run(
ex,
debug,
subject_id,
constant_memory,
data_zero_init,
max_epochs,
set_distribution_to_empirical,
ot_on_class_dims,
independent_class_dists,
half_before,
n_sensors,
final_hz,
start_ms,
stop_ms,
model_name,
final_fft,
clf_loss,
save_model,
only_return_exp,
):
kwargs = locals()
kwargs.pop("ex")
kwargs.pop("only_return_exp")
kwargs.pop("save_model")
th.backends.cudnn.benchmark = True
import sys
logging.basicConfig(
format="%(asctime)s %(levelname)s : %(message)s",
level=logging.DEBUG,
stream=sys.stdout,
)
start_time = time.time()
ex.info["finished"] = False
confirm_gpu_availability()
epochs_df, feature_model, class_dist = run_exp(**kwargs)
end_time = time.time()
run_time = end_time - start_time
ex.info["finished"] = True
last_row = epochs_df.iloc[-1]
for key, val in last_row.iteritems():
ex.info[key] = float(val)
ex.info["runtime"] = run_time
save_pkl_artifact(ex, epochs_df, "epochs_df.pkl")
if save_model:
save_torch_artifact(ex, feature_model, "feature_model.pkl")
save_torch_artifact(ex, class_dist, "class_dist.pkl")
print("Finished!")
def run(
ex,
debug,
subject_id,
max_epochs,
n_sensors,
final_hz,
half_before,
start_ms,
stop_ms,
model,
save_model,
weight_decay,
only_return_exp,
final_fft,
add_bnorm,
act_norm,
):
kwargs = locals()
kwargs.pop("ex")
kwargs.pop("only_return_exp")
kwargs.pop("save_model")
th.backends.cudnn.benchmark = True
import sys
logging.basicConfig(
format="%(asctime)s %(levelname)s : %(message)s",
level=logging.DEBUG,
stream=sys.stdout,
)
start_time = time.time()
ex.info["finished"] = False
confirm_gpu_availability()
epochs_df, model = run_exp(**kwargs)
end_time = time.time()
run_time = end_time - start_time
ex.info["finished"] = True
last_row = epochs_df.iloc[-1]
for key, val in last_row.iteritems():
ex.info[key] = float(val)
ex.info["runtime"] = run_time
save_pkl_artifact(ex, epochs_df, "epochs_df.pkl")
if save_model:
save_torch_artifact(ex, model, "model.pkl")
print("Finished!")
def run(ex, test_on_eval, sensor_types, n_chans, max_recording_mins,
n_recordings, sec_to_cut_at_start, sec_to_cut_at_end,
duration_recording_mins, test_recording_mins, max_abs_val,
clip_before_resample, sampling_freq, divisor, n_folds, i_test_fold,
shuffle, merge_train_valid, model_name, input_time_length,
final_conv_length, stride_before_pool, n_start_chans, n_chan_factor,
optimizer, learning_rate, weight_decay, scheduler, model_constraint,
batch_size, max_epochs, save_predictions, save_crop_predictions,
np_th_seed, only_return_exp):
log_dir = ex.observers[0].dir
kwargs = locals()
kwargs.pop('ex')
kwargs.pop('save_predictions')
kwargs.pop('save_crop_predictions')
import sys
logging.basicConfig(format='%(asctime)s %(levelname)s : %(message)s',
level=logging.DEBUG,
stream=sys.stdout)
start_time = time.time()
ex.info['finished'] = False
confirm_gpu_availability()
exp = run_exp(**kwargs)
end_time = time.time()
run_time = end_time - start_time
ex.info['finished'] = True
if not only_return_exp:
last_row = exp.epochs_df.iloc[-1]
for key, val in last_row.iteritems():
ex.info[key] = float(val)
ex.info['runtime'] = run_time
if not only_return_exp:
save_pkl_artifact(ex, exp.epochs_df, 'epochs_df.pkl')
save_pkl_artifact(ex, exp.before_stop_df, 'before_stop_df.pkl')
save_torch_artifact(ex, exp.model.state_dict(), 'model_params.pkl')
if save_predictions:
exp.model.eval()
for setname in ('train', 'valid', 'test'):
log.info(
"Compute and save predictions for {:s}...".format(setname))
dataset = exp.datasets[setname]
log.info("Save labels for {:s}...".format(setname))
save_npy_artifact(ex, dataset.y,
'{:s}_trial_labels.npy'.format(setname))
preds_per_batch = [
var_to_np(exp.model(np_to_var(b[0]).cuda()))
for b in exp.iterator.get_batches(dataset, shuffle=False)
]
preds_per_trial = compute_preds_per_trial(
preds_per_batch,
dataset,
input_time_length=exp.iterator.input_time_length,
n_stride=exp.iterator.n_preds_per_input)
mean_preds_per_trial = [
np.mean(preds, axis=(0, 2)) for preds in preds_per_trial
]
mean_preds_per_trial = np.array(mean_preds_per_trial)
log.info("Save trial predictions for {:s}...".format(setname))
save_npy_artifact(ex, mean_preds_per_trial,
'{:s}_trial_preds.npy'.format(setname))
if save_crop_predictions:
log.info(
"Save crop predictions for {:s}...".format(setname))
save_npy_artifact(ex, preds_per_trial,
'{:s}_crop_preds.npy'.format(setname))
else:
return exp