def test_create_polytomous_data_fail(self):
"""Testing synthetic polytomous function fails with 1 level"""
np.random.seed(808)
difficulty = np.array([[1.0]])
bad_graded_difficulty = np.random.rand(4, 5)
even_gum_difficulty = np.random.rand(4, 4)
with self.assertRaises(AssertionError):
create_synthetic_irt_polytomous(difficulty, difficulty, difficulty)
with self.assertRaises(KeyError):
create_synthetic_irt_polytomous([1, 2, 3],
difficulty,
difficulty,
model='boom')
with self.assertRaises(AssertionError):
create_synthetic_irt_polytomous(bad_graded_difficulty, 1.2,
np.random.randn(100))
with self.assertRaises(AssertionError):
create_synthetic_irt_polytomous(even_gum_difficulty,
1.2,
np.random.randn(100),
model="gum")
# Test for skew symmetric fail
with self.assertRaises(AssertionError):
create_synthetic_irt_polytomous(bad_graded_difficulty,
1.2,
np.random.randn(100),
model="gum")
def setUp(self):
np.random.seed(1944)
self.difficulty = np.random.randn(10, 4)
self.discrimination = np.random.rand(10) + 0.5
thetas = np.random.randn(1000)
thetas_smol = thetas[:500].copy()
self.discrimination_smol = self.discrimination[:5].copy()
self.difficulty_smol = self.difficulty[:5, :].copy()
self.syn_data_smol = create_synthetic_irt_polytomous(
self.difficulty_smol,
self.discrimination_smol,
thetas_smol,
model='PCM',
seed=546)
self.syn_data_larg = create_synthetic_irt_polytomous(
self.difficulty,
self.discrimination,
thetas,
model='PCM',
seed=546)
self.syn_data_mixed = create_synthetic_irt_polytomous(
self.difficulty_smol[:, 1:],
self.discrimination_smol,
thetas,
model='PCm',
seed=543)
def test_check_polytomous_discrimination(self):
"""Smoke tests if a single value for discrimination passes"""
difficulty = np.array([[1.0, 2., 3.]])
discrimination = 3
# Simple Smoke tests
create_synthetic_irt_polytomous(difficulty, discrimination, difficulty)
create_synthetic_irt_polytomous(difficulty, np.array([discrimination]),
difficulty)
def test_check_polytomous_regression(self):
"""Regression testing graded and credit polytomous functions"""
seed = 876
np.random.seed(seed)
difficulty_gum = np.array([
np.linspace(-1.2, 1.2, 5),
np.linspace(-.8, .8, 5),
np.linspace(-1.7, 1.7, 5),
np.linspace(-1.0, 1.0, 5)
])
difficulty_pcm = np.random.randn(5, 4)
difficulty = np.sort(difficulty_pcm, axis=1)
discrimination = 1.23
thetas = np.random.randn(8)
# Regression Tests
poly_data_graded = create_synthetic_irt_polytomous(difficulty,
discrimination,
thetas,
model='grm',
seed=seed)
poly_data_credit = create_synthetic_irt_polytomous(difficulty_pcm,
discrimination,
thetas,
model='pcm',
seed=seed)
poly_data_gum = create_synthetic_irt_polytomous(difficulty_gum,
discrimination,
thetas,
model='gum',
seed=seed)
expected_graded = np.array([[5, 2, 2, 5, 2, 2, 5, 5],
[5, 5, 1, 5, 5, 1, 2, 1],
[5, 5, 1, 3, 2, 1, 5, 1],
[3, 5, 3, 5, 4, 1, 4, 1],
[5, 5, 3, 4, 3, 3, 5, 2]])
expected_partial = np.array([[5, 5, 2, 5, 2, 1, 5, 3],
[5, 5, 1, 5, 4, 1, 4, 1],
[5, 5, 1, 3, 1, 1, 5, 1],
[5, 5, 1, 5, 4, 1, 5, 1],
[5, 5, 2, 5, 4, 2, 5, 2]])
expected_unfold = np.array([[2, 1, 2, 2, 2, 2, 2, 3],
[2, 2, 1, 2, 3, 1, 1, 1],
[2, 3, 2, 2, 3, 1, 2, 1],
[1, 3, 3, 3, 3, 1, 1, 1]])
np.testing.assert_array_equal(poly_data_graded, expected_graded)
np.testing.assert_array_equal(poly_data_credit, expected_partial)
np.testing.assert_array_equal(poly_data_gum, expected_unfold)
def test_gum_mml(self):
"""Testing unfolding response mml/jml for missing data."""
np.random.seed(7382)
n_items = 15
n_people = 600
discrimination = np.sqrt(-2 * np.log(np.random.rand(n_items)))
delta = np.random.randn(n_items, 1)
difficulty = np.random.randn(n_items, 3)
difficulty = np.sort(difficulty, 1)
difficulty = np.c_[-difficulty,
np.zeros((n_items, 1)), difficulty[:, ::-1]] + delta
theta = np.random.randn(n_people)
syn_data = create_synthetic_irt_polytomous(difficulty,
discrimination,
theta,
model='gum')
syn_data_tagged = _create_missing_data(syn_data, 112, 0.1)
syn_data_missing = tag_missing_data(syn_data_tagged, [1, 2, 3, 4])
# MML
result_all_good = gum_mml(syn_data)
result_missing = gum_mml(syn_data_missing)
difference_rmse = _rmse(result_all_good['Delta'],
result_missing['Delta'])
self.assertAlmostEqual(difference_rmse, 0.0664, 3)
difference_rmse = _rmse(result_all_good['Discrimination'],
result_missing['Discrimination'])
self.assertAlmostEqual(difference_rmse, 0.1382, 2)
def test_unfolding_run(self):
"""Testing the unfolding model runs."""
np.random.seed(555)
difficulty = -(np.random.rand(10, 2) * 1.5)
delta = 0.5 * np.random.rand(10, 1)
discrimination = np.random.rand(10) + 0.5
thetas = np.random.randn(200)
betas = np.c_[difficulty, np.zeros((10, 1)), -difficulty[:, ::-1]]
betas += delta
syn_data = create_synthetic_irt_polytomous(betas,
discrimination,
thetas,
model='GUM',
seed=546)
result = gum_mml(syn_data, {'max_iteration': 100})
rmse_discrimination = np.sqrt(
np.square(discrimination.squeeze() - result[0]).mean())
rmse_delta = np.sqrt(np.square(delta.squeeze() - result[1]).mean())
rmse_tau = np.sqrt(np.square(difficulty - result[2]).mean())
self.assertAlmostEqual(rmse_discrimination, 0.3709931291, places=5)
self.assertAlmostEqual(rmse_delta, 1.18368786, places=5)
self.assertAlmostEqual(rmse_tau, 0.6406162162, places=5)
def test_graded_small_participant(self):
"""Regression Testing graded response model with small N."""
np.random.seed(87)
difficulty = np.sort(np.random.randn(5, 4), axis=1)
discrimination = np.random.rand(5) + 0.5
thetas = np.random.randn(101)
syn_data = create_synthetic_irt_polytomous(difficulty, discrimination,
thetas)
estimated_parameters = grm_mml(syn_data)
# Regression test
expected_discrimination = np.array(
[1.72485717, 0.39305266, 0.82841429, 0.93731447, 1.56774651])
expectected_difficulty = np.array(
[[-0.1571751, 0.22757599, 0.33438484, 0.59311136],
[1.22266539, 1.3332354, 2.26915694, 3.51488207],
[-1.25515215, -1.06235363, -0.70073232, 0.87821931],
[-1.26630059, -0.37880122, 2.25720247, np.nan],
[-1.36961213, -0.66186681, -0.50308306, np.nan]])
np.testing.assert_array_almost_equal(estimated_parameters[0],
expected_discrimination)
np.testing.assert_array_almost_equal(estimated_parameters[1],
expectected_difficulty)
def test_graded_large_participant(self):
"""Regression Testing graded response model with large N."""
np.random.seed(1944)
difficulty = np.sort(np.random.randn(5, 4), axis=1)
discrimination = np.random.rand(5) + 0.5
thetas = np.random.randn(600)
syn_data = create_synthetic_irt_polytomous(difficulty, discrimination,
thetas)
estimated_parameters = grm_mml(syn_data)
# Regression test
expected_discrimination = np.array(
[0.58291387, 1.38349382, 0.87993092, 1.17329774, 1.47195824])
expectected_difficulty = np.array(
[[-1.23881672, -0.5340817, 0.06170343, 1.23881201],
[-1.09279868, -0.76747967, 0.44660955, 1.28909032],
[-0.16828803, 0.23943693, 0.76140209, 1.24435541],
[-2.02935022, -0.70214267, -0.23281603, 1.27521818],
[-1.47758497, -0.9050062, 0.0698804, 0.71286592]])
np.testing.assert_array_almost_equal(estimated_parameters[0],
expected_discrimination)
np.testing.assert_array_almost_equal(estimated_parameters[1],
expectected_difficulty)
def test_grm_jml_mml(self):
"""Testing graded response mml/jml for missing data."""
np.random.seed(69871)
n_items = 10
n_people = 300
discrimination = np.sqrt(-2 * np.log(np.random.rand(n_items)))
difficulty = np.random.randn(n_items, 3)
difficulty = np.sort(difficulty, 1)
theta = np.random.randn(n_people)
syn_data = create_synthetic_irt_polytomous(difficulty,
discrimination,
theta,
model='grm')
syn_data_tagged = _create_missing_data(syn_data, 879858, 0.1)
syn_data_missing = tag_missing_data(syn_data_tagged, [1, 2, 3, 4])
# JML
result_all_good = grm_jml(syn_data)
result_missing = grm_jml(syn_data_missing)
difference_rmse = _rmse(result_all_good['Difficulty'].ravel(),
result_missing['Difficulty'].ravel())
self.assertAlmostEqual(difference_rmse, 0.097251, 4)
difference_rmse = _rmse(result_all_good['Discrimination'],
result_missing['Discrimination'])
self.assertAlmostEqual(difference_rmse, 0.112861, 4)
# MML
result_all_good = grm_mml(syn_data)
result_missing = grm_mml(syn_data_missing)
difference_rmse = _rmse(result_all_good['Difficulty'].ravel(),
result_missing['Difficulty'].ravel())
self.assertAlmostEqual(difference_rmse, 0.088626, 4)
difference_rmse = _rmse(result_all_good['Discrimination'],
result_missing['Discrimination'])
self.assertAlmostEqual(difference_rmse, 0.065834, 4)
# EAP/MML
result_all_good = grm_mml_eap(syn_data)
result_missing = grm_mml_eap(syn_data_missing)
difference_rmse = _rmse(result_all_good['Difficulty'].ravel(),
result_missing['Difficulty'].ravel())
self.assertAlmostEqual(difference_rmse, 0.073556, 4)
difference_rmse = _rmse(result_all_good['Discrimination'],
result_missing['Discrimination'])
self.assertAlmostEqual(difference_rmse, 0.062288, 4)
def test_graded_response_model_close(self):
"""Regression Testing graded response model with large N."""
np.random.seed(6322)
difficulty = np.sort(np.random.randn(10, 4), axis=1)
discrimination = np.random.rand(10) + 0.5
thetas = np.random.randn(1000)
syn_data = create_synthetic_irt_polytomous(difficulty, discrimination,
thetas)
estimated_parameters = grm_mml(syn_data)
rmse = np.sqrt(
np.square(estimated_parameters[0] - discrimination).mean())
self.assertLess(rmse, .0966)
rmse = np.sqrt(np.square(estimated_parameters[1] - difficulty).mean())
self.assertLess(rmse, .1591)
def test_grm_mml_eap_method(self):
"""Testing the GRM EAP/MML Method."""
np.random.seed(99854)
n_items = 10
n_people = 300
difficulty = stats.norm(0, 1).rvs(n_items * 3).reshape(n_items, -1)
difficulty = np.sort(difficulty, axis=1)
discrimination = stats.rayleigh(loc=0.25, scale=.8).rvs(n_items)
thetas = np.random.randn(n_people)
syn_data = create_synthetic_irt_polytomous(difficulty, discrimination,
thetas)
result = grm_mml_eap(syn_data, {'hyper_quadrature_n': 21})
# Smoke Tests / Regression Tests
expected_difficulty = np.array(
[[-0.68705911, 0.25370937, 0.62872705],
[-1.81331475, -1.52607597, 0.01957819],
[-2.16305964, -0.51648053, 0.20447022],
[-1.51064069, -1.18709807, 1.74368598],
[-2.44714587, -1.01438472, -0.44406173],
[-1.38622596, -0.11417447, 1.14001425],
[-0.92724279, -0.11335446, 1.30273993],
[-0.55972331, -0.28527674, 0.01131112],
[-1.72941028, -0.34732405, 1.17681916],
[-1.73346085, -0.12292641, 0.91797906]])
expected_discrimination = np.array([
1.35572245, 0.77018004, 0.92848851, 1.6339604, 0.79229545,
2.35881697, 0.64452994, 1.86795956, 1.56986454, 1.93426233
])
expected_rayleigh_scale = 0.9161607303681261
np.testing.assert_allclose(result['Difficulty'],
expected_difficulty,
atol=1e-3,
rtol=1e-3)
np.testing.assert_allclose(result['Discrimination'],
expected_discrimination,
atol=1e-3,
rtol=1e-3)
self.assertAlmostEqual(result['Rayleigh_Scale'],
expected_rayleigh_scale, 3)
def test_graded_jml_regression(self):
"""Testing joint maximum grm model."""
np.random.seed(1022)
difficulty = np.sort(np.random.randn(5, 3), axis=1)
discrimination = 0.5 + np.random.rand(5)
thetas = np.random.randn(50)
syn_data = create_synthetic_irt_polytomous(difficulty, discrimination,
thetas)
output = grm_jml(syn_data)
# Expected Outputs (Basically a smoke test)
alphas = np.array([4., 0.79558366, 0.25, 4., 1.84876057])
betas = np.array([[-0.06567396, 0.00834646, 0.04343122],
[-1.72554323, -1.56602067, -0.87385678],
[-3.30647782, 1.86102210, np.nan],
[-0.47923614, 0.31797999, 0.89676892],
[-0.67769087, 0.49737400, np.nan]])
np.testing.assert_allclose(alphas, output[0], rtol=1e-5)
np.testing.assert_allclose(betas, output[1], rtol=1e-5)
def test_pcm_jml_mml(self):
"""Testing partial response mml/jml for missing data."""
np.random.seed(499867)
n_items = 10
n_people = 300
discrimination = np.sqrt(-2 * np.log(np.random.rand(n_items)))
difficulty = np.random.randn(n_items, 3)
difficulty = np.sort(difficulty, 1)
theta = np.random.randn(n_people)
syn_data = create_synthetic_irt_polytomous(difficulty,
discrimination,
theta,
model='pcm')
syn_data_tagged = _create_missing_data(syn_data, 1986778, 0.1)
syn_data_missing = tag_missing_data(syn_data_tagged, [1, 2, 3, 4])
# JML
result_all_good = pcm_jml(syn_data)
result_missing = pcm_jml(syn_data_missing)
difference_rmse = _rmse(result_all_good['Difficulty'].ravel(),
result_missing['Difficulty'].ravel())
self.assertAlmostEqual(difference_rmse, 0.078015, 4)
difference_rmse = _rmse(result_all_good['Discrimination'],
result_missing['Discrimination'])
self.assertAlmostEqual(difference_rmse, 0.147213, 4)
# MML
result_all_good = pcm_mml(syn_data)
result_missing = pcm_mml(syn_data_missing)
difference_rmse = _rmse(result_all_good['Difficulty'].ravel(),
result_missing['Difficulty'].ravel())
self.assertAlmostEqual(difference_rmse, 0.085013, 4)
difference_rmse = _rmse(result_all_good['Discrimination'],
result_missing['Discrimination'])
self.assertAlmostEqual(difference_rmse, 0.079640, 4)
def test_partial_credit_jml_regression(self):
"""Testing joint maximum partial credit model."""
np.random.seed(3)
difficulty = np.random.randn(5, 3)
discrimination = 0.5 + np.random.rand(5)
thetas = np.random.randn(50)
syn_data = create_synthetic_irt_polytomous(difficulty,
discrimination,
thetas,
model='pcm')
output = pcm_jml(syn_data)
# Expected Outputs (Basically a smoke test)
alphas = np.array([0.41826845, 4., 0.356021, 0.42943596, 4.])
betas = [[6., -1.83001522, 0.57618678],
[-1.34063596, -0.36478777, 0.3783891],
[3.32583095, -1.63422385, 0.93340261],
[2.58060883, -3.65355207, 1.80558368],
[0.55722762, 1.01035413, 0.74398657]]
np.testing.assert_allclose(alphas, output[0], atol=1e-4)
np.testing.assert_allclose(betas, output[1], atol=1e-4)
def test_partial_credit_jml_regression(self):
"""Testing joint maximum partial credit model."""
np.random.seed(3)
difficulty = np.random.randn(5, 3)
discrimination = 0.5 + np.random.rand(5)
thetas = np.random.randn(50)
syn_data = create_synthetic_irt_polytomous(difficulty,
discrimination,
thetas,
model='pcm')
output = pcm_jml(syn_data)
# Expected Outputs (Basically a smoke test)
alphas = np.array([0.41826845, 4., 0.356021, 0.429537, 4.])
betas = [[6., -1.83001497, 0.57618739],
[-1.34063642, -0.36478753, 0.3783893],
[3.32581876, -1.63421762, 0.93340153],
[2.57971531, -3.65201053, 1.80513887],
[0.55722782, 1.01035442, 0.74398655]]
np.testing.assert_allclose(alphas, output['Discrimination'], rtol=1e-3)
np.testing.assert_allclose(betas, output['Difficulty'], rtol=1e-3)