def validation_procedure(experiment_design, experiment_design_indep, experiment_design_one_mode, experiment_design_one_mode_indep, validate_on_network=True): # Get the validation data experiment_name = experiment_design['name'] # We will load the test data from this experiment experiment_name_one_mode = experiment_design_one_mode['name'] # Get the network basic_geometry = experiment_design['basic_geometry'] # Nearly everything will need a network. net = get_network(**basic_geometry) # reload the HMM and the GMRF estimator from the files # All we need for testing is a experiment_design test_hmm = read_hmm_pickle(experiment_name) # Read the estimator gmrf_estimation = experiment_design['gmrf_estimation'] test_gmrf_estimator = get_gmrf_estimator(experiment_name, gmrf_estimation['process']) # Baseline Gaussian independent test_hmm_one_mode = read_hmm_pickle('{0}_one_mode'.format(experiment_name)) test_gmrf_one_mode_indep = get_gmrf_estimator('{0}_one_mode_indep'.format(experiment_name), 'diagonal') # Baseline Gaussian test_gmrf_one_mode = get_gmrf_estimator('{0}_one_mode'.format(experiment_name), gmrf_estimation['process']) # Baseline MultiMode Gaussian independent test_gmrf_indep = get_gmrf_estimator('{0}_indep'.format(experiment_name), 'diagonal') tic('Validation') test_traj_obs_all = list(test_traj_obs(experiment_name)) test_traj_obs_one_mode_all = list(test_traj_obs(experiment_name_one_mode)) tic("Validation set: {0} trajectories".format(len(test_traj_obs_all))) model = [(test_traj_obs_all, test_gmrf_estimator, test_hmm, 'MM-GMRF')] baseline1 = [(test_traj_obs_one_mode_all, test_gmrf_one_mode_indep, test_hmm_one_mode, 'one mode indep')] baseline2 = [(test_traj_obs_one_mode_all, test_gmrf_one_mode, test_hmm_one_mode, 'one mode')] baseline3 = [(test_traj_obs_all, test_gmrf_indep, test_hmm, 'multi-modal indep')] val_model = model + baseline1 + baseline2 + baseline3 tic('path validation') validate.validate_on_paths(val_model, net, estimation_sampling_process=experiment_design['estimation_sampling']['process'], estimation_sampling_parameters=experiment_design['estimation_sampling']['parameters'], **experiment_design['evaluation']) if validate_on_network: tic('network validation') validate.validate_on_network(val_model, net, estimation_sampling_process=experiment_design['estimation_sampling']['process'], estimation_sampling_parameters=experiment_design['estimation_sampling']['parameters'], **experiment_design['evaluation']) tic("Evaluation finished")
# Get the validation data test_dates = data_source['test_dates'] test_tspots_seqs = [ttob_seq for date in test_dates for ttob_seq in getDayTSpots(date, net)] test_traj_obs = [traj_ob for tspots_seq in test_tspots_seqs for traj_ob in traj_conv.mapTrajectory(tspots_seq, **traj_conv_param)] test_traj_obs_one_mode = [traj_ob for tspots_seq in test_tspots_seqs for traj_ob in traj_conv_one_mode.mapTrajectory(tspots_seq, **traj_conv_param)] test_traj_obs = traj_obs test_traj_obs_one_mode = traj_obs_one_mode # reload the HMM and the GMRF estimator from the files # All we need for testing is a experiment_design test_hmm = read_hmm(experiment_name) # Read the estimator test_gmrf_estimator = get_gmrf_estimator(experiment_name, gmrf_estimation['process']) # Baseline Gaussian independent test_hmm_one_mode = read_hmm('{0}_one_mode'.format(experiment_name)) test_gmrf_one_mode_indep = get_gmrf_estimator('{0}_one_mode_indep'.format(experiment_name), 'diagonal') # Baseline Gaussian test_gmrf_one_mode = get_gmrf_estimator('{0}_one_mode'.format(experiment_name), gmrf_estimation['process']) # Baseline MultiMode Gaussian independent test_gmrf_indep = get_gmrf_estimator('{0}_indep'.format(experiment_name), 'diagonal') # test_gmrf_estimator_exact = get_gmrf_estimator(experiment_name,{'process':'exact'}) # test_gmrf_estimator_diagonal = get_gmrf_estimator(experiment_name,{'process':'diagonal'}) # TODO: perform cross validate_on_paths # This is after stop/go, so it is easier than pure new paths # test_traj_obs = test_traj_obs[1:10]