def validation_procedure(experiment_design,
experiment_design_indep,
experiment_design_one_mode,
experiment_design_one_mode_indep,
validate_on_network=True):
# Get the validation data
experiment_name = experiment_design['name']
# We will load the test data from this experiment
experiment_name_one_mode = experiment_design_one_mode['name']
# Get the network
basic_geometry = experiment_design['basic_geometry']
# Nearly everything will need a network.
net = get_network(**basic_geometry)
# reload the HMM and the GMRF estimator from the files
# All we need for testing is a experiment_design
test_hmm = read_hmm_pickle(experiment_name)
# Read the estimator
gmrf_estimation = experiment_design['gmrf_estimation']
test_gmrf_estimator = get_gmrf_estimator(experiment_name, gmrf_estimation['process'])
# Baseline Gaussian independent
test_hmm_one_mode = read_hmm_pickle('{0}_one_mode'.format(experiment_name))
test_gmrf_one_mode_indep = get_gmrf_estimator('{0}_one_mode_indep'.format(experiment_name), 'diagonal')
# Baseline Gaussian
test_gmrf_one_mode = get_gmrf_estimator('{0}_one_mode'.format(experiment_name), gmrf_estimation['process'])
# Baseline MultiMode Gaussian independent
test_gmrf_indep = get_gmrf_estimator('{0}_indep'.format(experiment_name), 'diagonal')
tic('Validation')
test_traj_obs_all = list(test_traj_obs(experiment_name))
test_traj_obs_one_mode_all = list(test_traj_obs(experiment_name_one_mode))
tic("Validation set: {0} trajectories".format(len(test_traj_obs_all)))
model = [(test_traj_obs_all, test_gmrf_estimator, test_hmm, 'MM-GMRF')]
baseline1 = [(test_traj_obs_one_mode_all, test_gmrf_one_mode_indep, test_hmm_one_mode, 'one mode indep')]
baseline2 = [(test_traj_obs_one_mode_all, test_gmrf_one_mode, test_hmm_one_mode, 'one mode')]
baseline3 = [(test_traj_obs_all, test_gmrf_indep, test_hmm, 'multi-modal indep')]
val_model = model + baseline1 + baseline2 + baseline3
tic('path validation')
validate.validate_on_paths(val_model, net,
estimation_sampling_process=experiment_design['estimation_sampling']['process'],
estimation_sampling_parameters=experiment_design['estimation_sampling']['parameters'], **experiment_design['evaluation'])
if validate_on_network:
tic('network validation')
validate.validate_on_network(val_model, net,
estimation_sampling_process=experiment_design['estimation_sampling']['process'],
estimation_sampling_parameters=experiment_design['estimation_sampling']['parameters'], **experiment_design['evaluation'])
tic("Evaluation finished")
# Get the validation data
test_dates = data_source['test_dates']
test_tspots_seqs = [ttob_seq for date in test_dates
for ttob_seq in getDayTSpots(date, net)]
test_traj_obs = [traj_ob for tspots_seq in test_tspots_seqs
for traj_ob in traj_conv.mapTrajectory(tspots_seq, **traj_conv_param)]
test_traj_obs_one_mode = [traj_ob for tspots_seq in test_tspots_seqs
for traj_ob in traj_conv_one_mode.mapTrajectory(tspots_seq, **traj_conv_param)]
test_traj_obs = traj_obs
test_traj_obs_one_mode = traj_obs_one_mode
# reload the HMM and the GMRF estimator from the files
# All we need for testing is a experiment_design
test_hmm = read_hmm(experiment_name)
# Read the estimator
test_gmrf_estimator = get_gmrf_estimator(experiment_name, gmrf_estimation['process'])
# Baseline Gaussian independent
test_hmm_one_mode = read_hmm('{0}_one_mode'.format(experiment_name))
test_gmrf_one_mode_indep = get_gmrf_estimator('{0}_one_mode_indep'.format(experiment_name), 'diagonal')
# Baseline Gaussian
test_gmrf_one_mode = get_gmrf_estimator('{0}_one_mode'.format(experiment_name), gmrf_estimation['process'])
# Baseline MultiMode Gaussian independent
test_gmrf_indep = get_gmrf_estimator('{0}_indep'.format(experiment_name), 'diagonal')
# test_gmrf_estimator_exact = get_gmrf_estimator(experiment_name,{'process':'exact'})
# test_gmrf_estimator_diagonal = get_gmrf_estimator(experiment_name,{'process':'diagonal'})
# TODO: perform cross validate_on_paths
# This is after stop/go, so it is easier than pure new paths
# test_traj_obs = test_traj_obs[1:10]
