def setUp(self): """Sets up synthetic data to use for estimation.""" np.random.seed(7) self.difficulty = np.linspace(-2.4, 1.7, 15) self.discrimination = 0.5 + np.random.rand(15) * 2 self.discrimination_single = 1.702 self.expected_theta = np.random.randn(300) distribution = skewnorm(0.5, 0.2, 1.1) self.expected_skew = distribution.rvs(size=300) self.skew_expected_theta_func = distribution.pdf # Create first synthetic data test self.set_one = create_synthetic_irt_dichotomous(self.difficulty, self.discrimination, self.expected_theta, seed=312) # Create second synthetic data test self.set_two = create_synthetic_irt_dichotomous( self.difficulty, self.discrimination_single, self.expected_theta, seed=547) # Create Skewed data set self.set_three = create_synthetic_irt_dichotomous(self.difficulty, self.discrimination, self.expected_skew, seed=872) # Add missing values dataset = create_synthetic_irt_dichotomous(self.difficulty, self.discrimination, self.expected_theta, seed=312) mask = np.random.rand(*dataset.shape) < 0.1 dataset[mask] = INVALID_RESPONSE self.set_four = dataset # Regression Test self.regression_difficulty = np.linspace(-1.5, 1.5, 5) self.regression_discrimination = np.linspace(0.8, 1.8, 5) self.regression_theta = np.random.randn(10) self.set_five = create_synthetic_irt_dichotomous( self.regression_difficulty, self.regression_discrimination, self.regression_theta, seed=422)
def test_twopl_jml_mml(self): """Testing twopl mml/jml for missing data.""" np.random.seed(15335) n_items = 10 n_people = 200 discrimination = np.sqrt(-2 * np.log(np.random.rand(n_items))) difficulty = np.random.randn(n_items) theta = np.random.randn(n_people) syn_data = create_synthetic_irt_dichotomous(difficulty, discrimination, theta) syn_data_tagged = _create_missing_data(syn_data, 84433, 0.1) syn_data_missing = tag_missing_data(syn_data_tagged, [0, 1]) # JML result_all_good = twopl_jml(syn_data) result_missing = twopl_jml(syn_data_missing) difference_rmse = _rmse(result_all_good['Difficulty'], result_missing['Difficulty']) self.assertAlmostEqual(difference_rmse, 0.283604, 4) difference_rmse = _rmse(result_all_good['Discrimination'], result_missing['Discrimination']) self.assertAlmostEqual(difference_rmse, 0.669763, 4) # MML result_all_good = twopl_mml(syn_data, {'use_LUT': False}) result_missing = twopl_mml(syn_data_missing, {'use_LUT': False}) difference_rmse = _rmse(result_all_good['Difficulty'], result_missing['Difficulty']) self.assertAlmostEqual(difference_rmse, 0.064496, 4) difference_rmse = _rmse(result_all_good['Discrimination'], result_missing['Discrimination']) self.assertAlmostEqual(difference_rmse, 0.155868, 4)
def test_onepl_jml_mml(self): """Testing oneple mml/jml for missing data.""" np.random.seed(789595) n_items = 10 n_people = 200 discrimination = 1.53 difficulty = np.random.randn(n_items) theta = np.random.randn(n_people) syn_data = create_synthetic_irt_dichotomous(difficulty, discrimination, theta) syn_data_tagged = _create_missing_data(syn_data, 66877, 0.1) syn_data_missing = tag_missing_data(syn_data_tagged, [0, 1]) # JML result_all_good = onepl_jml(syn_data) result_missing = onepl_jml(syn_data_missing) difference_rmse = _rmse(result_all_good['Difficulty'], result_missing['Difficulty']) self.assertAlmostEqual(difference_rmse, 0.048643, 4) difference_rmse = _rmse(result_all_good['Discrimination'], result_missing['Discrimination']) self.assertAlmostEqual(difference_rmse, 0.017526, 4) # MML result_all_good = onepl_mml(syn_data) result_missing = onepl_mml(syn_data_missing) difference_rmse = _rmse(result_all_good['Difficulty'], result_missing['Difficulty']) self.assertAlmostEqual(difference_rmse, 0.031971, 4) difference_rmse = _rmse(result_all_good['Discrimination'], result_missing['Discrimination']) self.assertAlmostEqual(difference_rmse, 0.045726, 4)
def test_rasch_jml_mml(self): """Testing rasch mml/jml for missing data.""" np.random.seed(7321) n_items = 10 n_people = 200 difficulty = np.random.randn(n_items) theta = np.random.randn(n_people) syn_data = create_synthetic_irt_dichotomous(difficulty, 1.0, theta) syn_data_tagged = _create_missing_data(syn_data, 73435, 0.1) syn_data_missing = tag_missing_data(syn_data_tagged, [0, 1]) # JML result_all_good = rasch_jml(syn_data) result_missing = rasch_jml(syn_data_missing) difference_rmse = _rmse(result_all_good['Difficulty'], result_missing['Difficulty']) self.assertAlmostEqual(difference_rmse, 0.072384, 4) # MML result_all_good = rasch_mml(syn_data) result_missing = rasch_mml(syn_data_missing) difference_rmse = _rmse(result_all_good['Difficulty'], result_missing['Difficulty']) self.assertAlmostEqual(difference_rmse, 0.054726, 4)
def test_2pl_mml_eap_method(self): """Testing the 2PL EAP/MML Method.""" np.random.seed(618331) n_items = 5 n_people = 150 difficulty = stats.norm(0, 1).rvs(n_items) discrimination = stats.rayleigh(loc=0.25, scale=.8).rvs(n_items) thetas = np.random.randn(n_people) syn_data = create_synthetic_irt_dichotomous(difficulty, discrimination, thetas) result = twopl_mml_eap(syn_data, {'hyper_quadrature_n': 21}) # Smoke Tests / Regression Tests expected_difficulty = np.array( [-0.2436698, 0.66299148, 1.3451037, -0.68059041, 0.40516614]) expected_discrimination = np.array( [1.99859796, 0.67420679, 1.18591025, 1.60937911, 1.19672389]) expected_rayleigh_scale = 0.9106036068099617 np.testing.assert_allclose(result['Difficulty'], expected_difficulty, atol=1e-3, rtol=1e-3) np.testing.assert_allclose(result['Discrimination'], expected_discrimination, atol=1e-3, rtol=1e-3) self.assertAlmostEqual(result['Rayleigh_Scale'], expected_rayleigh_scale, 3)
def test_2pl_mml_eap_method_csirt(self): """Testing the 2PL EAP/MML Method with CSIRT.""" np.random.seed(779841) n_items = 10 n_people = 300 difficulty = stats.norm(0, 1).rvs(n_items) discrimination = stats.rayleigh(loc=0.25, scale=.8).rvs(n_items) thetas = np.random.randn(n_people) syn_data = create_synthetic_irt_dichotomous(difficulty, discrimination, thetas) result = twopl_mml_eap(syn_data, {'estimate_distribution': True}) # Smoke Tests / Regression Tests expected_difficulty = np.array([ -0.91561408, 1.29631473, 1.01751178, -0.10536047, -0.02235909, -0.56510317, -1.67564893, -1.45646904, 1.89544833, -0.78602385 ]) expected_discrimination = np.array([ 0.9224411, 0.88102312, 0.86716565, 1.38012222, 0.67176012, 1.84035622, 1.58453053, 1.11488035, 1.07633054, 1.44767879 ]) expected_rayleigh_scale = 0.7591785686427288 np.testing.assert_allclose(result['Difficulty'], expected_difficulty, atol=1e-3, rtol=1e-3) np.testing.assert_allclose(result['Discrimination'], expected_discrimination, atol=1e-3, rtol=1e-3) self.assertAlmostEqual(result['Rayleigh_Scale'], expected_rayleigh_scale, 3)
def setUp(self): """Setup synthetic data for tests.""" np.random.seed(873) difficulty = np.linspace(-1.5, 1.5, 5) discrimination = 1.843 thetas = np.random.randn(600) syn_data = create_synthetic_irt_dichotomous(difficulty, discrimination, thetas) self.data = syn_data self.discrimination = discrimination
def test_rasch_close(self): """Testing rasch converging methods.""" np.random.seed(333) difficulty = np.linspace(-1.25, 1.25, 5) discrimination = 0.87 thetas = np.random.randn(2000) syn_data = create_synthetic_irt_dichotomous(difficulty, discrimination, thetas) output = rasch_mml(syn_data, discrimination) np.testing.assert_array_almost_equal(difficulty, output, decimal=1)
def setUp(self): """Setup synthetic data for tests.""" np.random.seed(846) difficulty = np.linspace(-1.5, 1.5, 5) discrimination = np.random.rand(5) + 0.5 thetas = np.random.randn(600) guessing = np.random.rand(5) * .3 syn_data = create_synthetic_irt_dichotomous(difficulty, discrimination, thetas, guessing=guessing) self.data = syn_data self.guessing = guessing
def test_conditional_close(self): """Testing conditional rasch model for accuracy.""" np.random.seed(574) difficuly = np.linspace(-1.5, 1.5, 5) discrimination = 1.2 thetas = np.random.randn(1600) syn_data = create_synthetic_irt_dichotomous(difficuly, discrimination, thetas) output = rasch_conditional(syn_data, discrimination)['Difficulty'] np.testing.assert_array_almost_equal(difficuly, output, decimal=1)
def test_onepl_close(self): """Testing onepl converging methods.""" np.random.seed(843) difficulty = np.linspace(-1.25, 1.25, 10) discrimination = 0.87 thetas = np.random.randn(1000) syn_data = create_synthetic_irt_dichotomous(difficulty, discrimination, thetas) output = onepl_mml(syn_data) self.assertLess(np.abs(output[0] - discrimination).max(), 0.1) self.assertLess(np.abs(output[1] - difficulty).max(), 0.2)
def test_conditional_regression_discrimination(self): """Testing conditional rasch model with non-unity discrimination.""" np.random.seed(142) difficuly = np.linspace(-1.5, 1.5, 5) discrimination = 1.7 thetas = np.random.randn(600) syn_data = create_synthetic_irt_dichotomous(difficuly, discrimination, thetas) output = rasch_conditional(syn_data, discrimination)['Difficulty'] expected_output = np.array( [-1.38086088, -0.74781933, -0.01267694, 0.77906715, 1.36228999]) np.testing.assert_allclose(expected_output, output)
def test_conditional_regression(self): """Testing conditional rasch model.""" np.random.seed(91) difficuly = np.linspace(-1.5, 1.5, 5) discrimination = 1 thetas = np.random.randn(600) syn_data = create_synthetic_irt_dichotomous(difficuly, discrimination, thetas) output = rasch_conditional(syn_data)['Difficulty'] expected_output = np.array( [-1.39893814, -0.80083855, -0.00947712, 0.61415543, 1.59509838]) np.testing.assert_allclose(expected_output, output)
def test_joint_regression_rasch(self): """Testing joint maximum rasch model.""" np.random.seed(91) difficuly = np.linspace(-1.5, 1.5, 5) discrimination = 1 thetas = np.random.randn(600) syn_data = create_synthetic_irt_dichotomous(difficuly, discrimination, thetas) output = rasch_jml(syn_data) expected_output = np.array( [-1.61394095, -0.88286827, 0.04830973, 0.77146166, 1.95939084]) np.testing.assert_allclose(expected_output, output)
def setUp(self): """Setup synthetic data for tests.""" np.random.seed(45184) difficulty = np.linspace(-1.5, 1.5, 10) discrimination = np.random.rand(10) + 0.5 thetas = np.random.randn(1000) guessing = np.random.rand(10) * .25 syn_data = create_synthetic_irt_dichotomous(difficulty, discrimination, thetas, guessing=guessing) self.data = syn_data self.guessing = guessing self.discrimination = discrimination self.difficulty = difficulty
def test_synthetic_irt_creation(self): """Testing the creation of synthetic data.""" seed = 31 # Regression test expected = np.array([[False, False, False, False, True, True], [False, False, True, True, True, True], [False, False, False, True, True, True]]) value = create_synthetic_irt_dichotomous(np.array([1.2, -0.2, 1.3]), 1.31, np.linspace(-6, 6, 6), seed=seed) np.testing.assert_array_equal(expected, value)
def test_joint_regression_onepl(self): """Testing joint maximum onepl model.""" np.random.seed(118) difficuly = np.linspace(-1.5, 1.5, 5) discrimination = 1.34 thetas = np.random.randn(600) syn_data = create_synthetic_irt_dichotomous(difficuly, discrimination, thetas) output = onepl_jml(syn_data) expected_output = np.array( [-1.70585219, -1.03551581, -0.09329877, 0.92320069, 1.77063402]) np.testing.assert_allclose(expected_output, output[1]) self.assertAlmostEqual(1.5316042, output[0])
def test_joint_regression_twopl(self): """Testing joint maximum twopl model.""" np.random.seed(138) difficuly = np.linspace(-1.5, 1.5, 5) discrimination = 0.5 + np.random.rand(5) thetas = np.random.randn(600) syn_data = create_synthetic_irt_dichotomous(difficuly, discrimination, thetas) output = twopl_jml(syn_data) # Expected Outputs alphas = np.array([0.29202081, 4., 0.91621924, 4., 0.27536785]) betas = np.array([-6., -0.39644246, -0.00862153, 0.3869096, 6.]) np.testing.assert_allclose(alphas, output[0]) np.testing.assert_allclose(betas, output[1], rtol=1e-5)
def test_integral_equations(self): """Tests solving for integral given a ratio.""" np.random.seed(786) theta = np.random.randn(50000) discrimination = 1.43 difficulty = np.array([-.4, .1, .5]) # Compare against dichotomous data syn_data = create_synthetic_irt_dichotomous( difficulty, discrimination, theta) n0 = np.count_nonzero(~syn_data, axis=1) n1 = np.count_nonzero(syn_data, axis=1) ratio = n1 / (n1 + n0) theta = _get_quadrature_points(61, -5, 5) distribution = np.exp(-np.square(theta) / 2) / np.sqrt(2 * np.pi) results = _solve_integral_equations( discrimination, ratio, distribution, theta) np.testing.assert_array_almost_equal(results, difficulty, decimal=2)