def test_solve_with_recorder(self): # Number of persons. P = 100 # Number of items. I = 20 # Number of latent ability dimensions (sub-scales). C = 1 # Using 2-PL model with fixed discrimination and no asymptote for all items. asym = 0 # 0.25 discrimination = 1 X, theta, b, c = sim.generate_dichotomous_responses( P, I, C, asymptote=asym, discrimination=discrimination) recorder = nirt.run_recorder.RunRecorder() solver = nirt.solver_refinement.SolverRefinement(X, c, recorder=recorder) theta_approx = solver.solve() assert recorder.theta.keys() == set([0, 4, 8, 16]), \ "unexpected recorder theta key set {}".format(recorder.theta.keys()) assert len(recorder.theta[0]) == 1 assert all( len(recorder.theta[r]) == 2 for r in recorder.theta.keys() if r != 0) assert recorder.irf.keys() == set([4, 8, 16]), \ "unexpected recorder IRF key set {}".format(recorder.irf.keys()) assert all( len(recorder.irf[r]) == 2 for r in recorder.irf.keys() if r != 0)
def test_generate_continuous_responses(self): np.random.seed(0) # Number of persons. P = 100 # Number of items. I = 20 # Number of latent ability dimensions (sub-scales). C = 1 # Using 2-PL model with fixed discrimination and no asymptote for all items. asym = 0 # 0.25 discrimination = 1 x, theta, b, c = sim.generate_dichotomous_responses( P, I, C, asymptote=asym, discrimination=discrimination, dichotomous=False) assert x.shape == (P, I) assert theta.shape == (P, C) assert b.shape == (I, ) assert c.shape == (I, )
def test_initial_guess(self): """Verifies that the histograms built from the initial theta guess approximate the exact IRFs to a reasonable degree. """ I = 20 C = 1 asym = 0 discrimination = 1 # Exact IRFs. model_irf = [ lambda t, i=i: nirt.simulate.simulate_data.three_pl_model( t, discrimination, b[i], asym) for i in range(I) ] num_bins = 5 method = "uniform-fixed" #"quantile" # "uniform" # Check that the norm of the difference between the approximate and exact IRF at the nodes is small enough for # all items, and that this error decreases when the sample size increases. num_p = 5 num_experiments = 10 e_mean = [0] * num_p for k, P in enumerate(100 * 4**np.arange(num_p)): e = [0] * num_experiments for experiment in range(num_experiments): X, theta, b, c = \ sim.generate_dichotomous_responses(P, I, C, asymptote=asym, discrimination=discrimination) # Initial guess for thetas. t = nirt.likelihood.initial_guess(X, c) # Build IRFs. xlim = [(min(t[:, ci]) - 2, max(t[:, ci]) + 2) for ci in range(C)] grid = [ nirt.grid.create_grid(t[:, ci], num_bins, method=method, xlim=xlim[ci]) for ci in range(C) ] irf = [ nirt.irf.ItemResponseFunction(grid[c[i]], X[:, i]) for i in range(I) ] # Calculate the scaled, weighted L2 norm of the error in the approximate IRF at the nodes. # Weight = bin count (so that all persons contribute the same weight to the norm: more # dense bins should count more). error = nirt.error.error_norm_by_item(model_irf, irf) #print("P {} {:.3f} +- {:.3f}".format(P, error.mean(), error.std())) e[experiment] = error.mean() e_mean[k] = np.mean(e) assert_array_almost_equal( e_mean, [0.06139, 0.03444, 0.02465, 0.01991, 0.01670], 5)
def test_solve_multidimensional_theta(self): # Number of persons. P = 100 # Number of items. I = 20 # Number of latent ability dimensions (sub-scales). C = 5 # Using 2-PL model with fixed discrimination and no asymptote for all items. asym = 0 # 0.25 discrimination = 1 X, theta, b, c = sim.generate_dichotomous_responses(P, I, C, asymptote=asym, discrimination=discrimination) solver = nirt.solver_sampled_simulated_annealing.SolverSampledSimulatedAnnealing(X, c) theta = solver.solve() print(theta) assert theta == 0
def test_solve_unidimensional_theta(self): # Number of persons. P = 100 # Number of items. I = 20 # Number of latent ability dimensions (sub-scales). C = 1 # Using 2-PL model with fixed discrimination and no asymptote for all items. asym = 0 # 0.25 discrimination = 1 X, theta, b, c = sim.generate_dichotomous_responses(P, I, C, asymptote=asym, discrimination=discrimination) solver = nirt.solver_mcmc.SolverMcmc(X, c) theta_approx = solver.solve() assert theta_approx.shape == theta.shape
def setUp(self) -> None: for handler in logging.root.handlers[:]: logging.root.removeHandler(handler) logging.basicConfig(level=logging.WARN, format="%(levelname)-8s %(message)s", datefmt="%a, %d %b %Y %H:%M:%S") np.random.seed(0) # Number of persons. self.P = 100 # Number of items. self.I = 20 # Number of latent ability dimensions (sub-scales). self.C = 1 # Using 2-PL model with fixed discrimination and no asymptote for all items. self.x, self.theta, self.b, self.c = \ sim.generate_dichotomous_responses(self.P, self.I, self.C, asymptote=0)
if __name__ == "__main__": for handler in logging.root.handlers[:]: logging.root.removeHandler(handler) logging.basicConfig(level=logging.INFO, format="%(levelname)-8s %(message)s", datefmt="%a, %d %b %Y %H:%M:%S") # Generate synthetic data, uni-dimensional latent trait. np.random.seed(0) P = 1000 I = 20 C = 1 asym = 0 discrimination = 1 X, theta, b, c = sim.generate_dichotomous_responses( P, I, C, asymptote=asym, discrimination=discrimination) # Algorithm parameters. # Aberration accentuation parameter. s = 2.0 # Only probabilities up to this value are considered for aberration graph links. p_max = 0.3 # A collusion group must correspond to at least this fraction of leaked items. min_item_fraction = 0.2 # Use IRT to estimate the probability that a person gets an item right. p, irf = calculate_irf_proabilities(X, c) # Check IRF accuracy (since this is synthetic data we can calculate this here). model = [ lambda x, b=b[i]: sim.three_pl_model(x, discrimination, b, asym) for i in range(I)