def mle_1_parameter(theta, r, f, a, b, MAX_ITERATIONS=10000):
'''One parameter (Rausch) logistic ICC model parameter estimation using MLE
From "Item Response Theory: Parameter Estimation Techniques"
Chapter 2.3, pages 40-45 (esp Eqn 2.20) and
Appendix A, pages 289-297 (port of A.3 on p.294)'''
theta, r, f = map(np.asanyarray, [theta, r, f]) # ensure these are arrays
p = r / f
for i in range(MAX_ITERATIONS):
print "iteration", i
P = np.squeeze(logistic(dev_ab(a, b, theta)))
W = P * (1 - P)
W[np.where(W < W_CUTOFF)] = np.nan # Delete any dud values
dLdb = np.nansum(r - f * P)
d2Ldb2 = -np.nansum(f * W)
print 'dLdb', dLdb, 'd2L / db2', d2Ldb2
# Plot the Log Likelihood & 1st&2nd derivatives
'''
import plt
bs = np.arange(-10, 10, 0.1)
Ps = [ logistic(dev_ab(a, b, theta))
for b in bs ]
L = [ log_likelihood(theta, r, f, P)
for P in Ps ]
dLdb = [ np.nansum(r - f * P)
for P in Ps ]
d2Ldb2 = [ np.nansum(f * P * (1 - P))
for P in Ps ]
plt.ioff()
plt.plot(bs, L)
plt.plot(bs, dLdb)
plt.plot(bs, d2Ldb2)
plt.show()
'''
# Db = rhs * np.linalg.inv(mat) # rhs = np.nansum(f * W * V), mat = np.nansum(f * W) V = (p - P) / W
Db = dLdb / d2Ldb2
b += Db
print 'b', b, 'Db', Db
if abs(b) > MAX_ZETA:
raise ConvergenceError("OUT OF BOUNDS ERROR ITERATION %i" % i)
if abs(Db) <= .05:
break
if i == MAX_ITERATIONS:
print "REACHED MAXIMUM NUMBER OF ITERATIONS"
P = logistic(dev_ab(a, b, theta))
chi2 = chiSquared(f, p, P)
return a, b, chi2
def mle_2_parameter(theta, r, f, zeta, lam, MAX_ITERATIONS=10000):
'''Two parameter logistic ICC model parameter estimation using MLE
From "Item Response Theory: Parameter Estimation Techniques"
Chapter 2.3, pages 40-45 (esp Eqn 2.20) and
Appendix A, pages 289-297 (port of A.3 on p.294)'''
theta, r, f = map(np.asanyarray, [theta, r, f]) # ensure these are arrays
p = r / f
for i in range(MAX_ITERATIONS):
print "iteration", i
P = np.squeeze(logistic(dev_zeta_lam(zeta, lam, theta)))
W = P * (1 - P)
W[np.where(W < W_CUTOFF)] = np.nan # Delete any dud values
V = (p - P) / W
fW = f * W
fWV = fW * V
fWV2 = fWV * V
fWth = fW * theta
fWth2 = fWth * theta
fWthV = fWth * V
if np.nansum(fW) <= 0:
raise ConvergenceError("OUT OF BOUNDS ERROR ITERATION %i" % i)
mat = np.nansum([[fW, fWth], [fWth, fWth2]], axis=-1)
rhs = np.nansum([[fWV], [fWthV]], axis=-1)
mat_det = np.linalg.det(mat)
if mat_det == 0:
raise ConvergenceError("Non-invertible matrix encountered %i" % i)
Dzeta, Dlam = np.dot(np.linalg.inv(mat), rhs).flatten()
if mat_det <= DM_CUTOFF:
break
zeta += Dzeta
lam += Dlam
if abs(zeta) > MAX_ZETA or abs(lam) > MAX_LAM:
raise ConvergenceError("OUT OF BOUNDS ERROR ITERATION %i" % i)
if abs(Dzeta) <= .05 and abs(Dlam) <= .05:
break
if i == MAX_ITERATIONS:
print "REACHED MAXIMUM NUMBER OF ITERATIONS"
#chi2 = np.nansum(fWV2) # sum of f[i] * W[i] * v[i]^2
P = logistic(dev_zeta_lam(zeta, lam, theta))
chi2 = chiSquared(f, p, P)
return zeta, lam, chi2
def param_eval(self, d, ss, hps):
"""
At distance dist, evaluate the prob of connection
"""
p = util.logistic(d, ss['mu'], ss['lambda'])
p = p * (hps['p_max'] - hps['p_min']) + hps['p_min']
return p
def param_eval(self, d, ss, hps):
"""
At distance dist, evaluate the prob of connection
"""
rate = util.logistic(d, ss['mu'], hps['lambda'])
rate = rate * (ss['rate_scale'] - hps['rate_min']) + hps['rate_min']
return rate
def bayesian_predict(self, x):
pred_var = np.sum(x * np.dot(x, self.covar), axis=1)
pred_mean = np.dot(x, self.weights_map)
kappa = (1 + np.pi * pred_var / 8)**(
-0.5) # as defined in Bishop's book chapter 4
bayes_predictions = logistic(pred_mean * kappa)
return bayes_predictions
def read_outputs(self):
last = self.values.shape[1] - 1
r = np.dot(self.values[:, last], self.hidden_output_axons)
r = util.logisticize_array(r)
for i in range(0, self.outputs):
r[i] = util.logistic(r[i])
return r
def param_eval(self, d, ss, hps):
"""
At distance dist, evaluate the prob of connection
"""
rate = util.logistic(d, ss['mu'], hps['lambda'])
rate = rate * (ss['rate_scale'] - hps['rate_min']) + hps['rate_min']
return rate
def param_eval(self, d, ss, hps):
"""
At distance dist, evaluate the prob of connection
"""
p = util.logistic(d, ss['mu'], ss['lambda'])
p = p * (hps['p_max'] - hps['p_min']) + hps['p_min']
return p
def logistic_ll_hessian(weights, x):
"""Note the Hessian does not depend on the true labels."""
output_probs = logistic(np.dot(x, weights))
hessian = np.zeros([x.shape[1]] * 2, dtype=np.float64)
for i in range(x.shape[0]):
hessian -= output_probs[i] * (1 - output_probs[i]) * np.outer(
x[i, :], x[i, :])
return hessian
def run_hidden(self):
for i in range(0, self.hidden_depth-1):
layer = self.values[:, i]
next_layer = self.values[:, i+1]
new_values = np.dot(layer, self.hidden_axons[i, :, :])
self.values[:, i + 1] = np.add(self.values[:, i+1], new_values)
for a in range(0, self.hidden_width):
self.values[a, i + 1] = util.logistic(self.values[a, i+1])
def sample_data(self, ss, hps):
"""
NOTE THIS ONLY SAMPLES FROM THE PARAM P and not from
suffstats.
"""
d = np.random.exponential(hps['mu_hp'])
p = util.logistic(d, ss['mu'], ss['lambda'])
p = p * (hps['p_max'] - hps['p_min']) + hps['p_min']
link = np.random.rand() < p
x = np.zeros(1, dtype=self.data_dtype())
x[0]['distance'] = d
x[0]['link'] = link
return x[0]
def sample_data(self, ss, hps):
"""
NOTE THIS ONLY SAMPLES FROM THE PARAM P and not from
suffstats.
"""
d = np.random.exponential(hps['mu_hp'])
p = util.logistic(d, ss['mu'], ss['lambda'])
p = p * (hps['p_max'] - hps['p_min']) + hps['p_min']
link = np.random.rand() < p
x = np.zeros(1, dtype=self.data_dtype())
x[0]['distance'] = d
x[0]['link'] = link
return x[0]
def sample_data(self, ss, hps):
"""
NOTE THIS ONLY SAMPLES FROM THE PARAM P and not from
suffstats.
"""
d = np.random.exponential(hps['mu_hp'])
rate = util.logistic(d, ss['mu'], hps['lambda'])
rate = rate * (ss['rate_scale'] - hps['rate_min']) + hps['rate_min']
count = np.random.poisson(rate)
x = np.zeros(1, dtype=self.data_dtype())
x[0]['distance'] = d
x[0]['link'] = count
return x[0]
def sample_data(self, ss, hps):
"""
NOTE THIS ONLY SAMPLES FROM THE PARAM P and not from
suffstats.
"""
d = np.random.exponential(hps['mu_hp'])
rate = util.logistic(d, ss['mu'], hps['lambda'])
rate = rate * (ss['rate_scale'] - hps['rate_min']) + hps['rate_min']
count = np.random.poisson(rate)
x = np.zeros(1, dtype=self.data_dtype())
x[0]['distance'] = d
x[0]['link'] = count
return x[0]
def fit_and_evaluate_gaussian(split,
cond=False,
use_image_space=False,
return_avg=True):
"""
Fits a gaussian to the train data and evaluates it on the given split.
:param split: the data split to evaluate on, must be 'trn', 'val', or 'tst'
:param cond: boolean, whether to fit a gaussian per conditional
:param use_image_space: bool, whether to report log probability in [0, 1] image space (only for cifar and mnist)
:param return_avg: bool, whether to return average log prob with std error, or all log probs
:return: average log probability & standard error, or all lop probs
"""
assert is_data_loaded(), 'Dataset hasn\'t been loaded'
# choose which data split to evaluate on
data_split = getattr(data, split, None)
if data_split is None:
raise ValueError('Invalid data split')
if cond:
comps = []
for i in range(data.n_labels):
idx = data.trn.labels == i
comp = pdfs.fit_gaussian(data.trn.x[idx])
comps.append(comp)
prior = np.ones(data.n_labels, dtype=float) / data.n_labels
model = pdfs.MoG(prior, xs=comps)
else:
model = pdfs.fit_gaussian(data.trn.x)
logprobs = model.eval(data_split.x)
if use_image_space:
assert data_name in ['mnist', 'cifar10']
z = util.logistic(data_split.x)
logprobs += data.n_dims * np.log(1 - 2 * data.alpha) - np.sum(
np.log(z) + np.log(1 - z), axis=1)
if return_avg:
avg_logprob = logprobs.mean()
std_err = logprobs.std() / np.sqrt(data_split.N)
return avg_logprob, std_err
else:
return logprobs
def JL(params):
count[0] += 1
# unpack the parameters
a, b, c = unpack_abc(A, B, C, params)
Pstar = logistic(dev_ab(a, b, theta))
P = scale_guessing(Pstar, c)
if c == 0 and num_params < 3:
Pstar = None # optimize :)
JLL = J(theta, r, f, P, a, b, c, Pstar)
if return_history:
abc_hist.append((a, b, c))
if not use_2nd:
return JLL
HLL = H(theta, r, f, P, a, b, c, Pstar)
return JLL, HLL
def JL(params):
count[0] += 1
# unpack the parameters
zeta, lam, c = unpack_zlc(ZETA, LAM, C, params)
Pstar = logistic(dev_zeta_lam(zeta, lam, theta))
P = scale_guessing(Pstar, c)
if c == 0 and num_params < 3:
Pstar = None # optimize :)
JLL = J(theta, r, f, P, zeta, lam, c, Pstar)
if return_history:
zlc_hist.append((zeta, lam, c))
if not use_2nd:
return JLL
HLL = H(theta, r, f, P, zeta, lam, c, Pstar)
return JLL, HLL
def derivative_L(abc):
_a, _b, _c = unpack_abc(a, b, c, abc) # unpack the parameters
Pstar = logistic(dev_ab(_a, _b, theta))
P = scale_guessing(Pstar, _c)
return DL(theta, r, f, P, _a, _b, _c, Pstar)
def plot_t1t1_params(fig, conn_and_dist, assign_vect, ss, hps, MAX_DIST=10,
model="LogisticDistance", MAX_CLASSES = 20):
"""
In the same order that we would plot the latent matrix, plot
the per-parameter properties
hps are per-relation hps
note, tragically, this wants the whole figure
"""
from mpl_toolkits.axes_grid1 import Grid
from matplotlib import pylab
assign_vect = np.array(assign_vect)
canon_assign_vect = util.canonicalize_assignment(assign_vect)
# create the mapping between existing and new
canon_to_old = {}
for i, v in enumerate(canon_assign_vect):
canon_to_old[v]= assign_vect[i]
CLASSES = np.sort(np.unique(canon_assign_vect))
CLASSN = len(CLASSES)
if CLASSN > MAX_CLASSES:
print "WARNING, TOO MANY CLASSES"
CLASSN = MAX_CLASSES
img_grid = Grid(fig, 111, # similar to subplot(111)
nrows_ncols = (CLASSN, CLASSN),
axes_pad = 0.1,
add_all=True,
share_all=True,
label_mode = 'L',
)
if "istance" not in model:
return
for c1i, c1_canon in enumerate(CLASSES[:MAX_CLASSES]):
for c2i, c2_canon in enumerate(CLASSES[:MAX_CLASSES]):
c1 = canon_to_old[c1_canon]
c2 = canon_to_old[c2_canon]
ax_pos = c1i * CLASSN + c2i
ax = img_grid[ax_pos]
nodes_1 = np.argwhere(assign_vect == c1).flatten()
nodes_2 = np.argwhere(assign_vect == c2).flatten()
conn_dist_hist = []
noconn_dist_hist = []
flatten_dist_val = []
assert len(nodes_1) > 0
assert len(nodes_2) > 0
for n1 in nodes_1:
for n2 in nodes_2:
d = conn_and_dist[n1, n2]['distance']
if conn_and_dist[n1, n2]['link']:
conn_dist_hist.append(d)
else:
noconn_dist_hist.append(d)
flatten_dist_val.append((d, conn_and_dist[n1, n2]['link']))
flatten_dist_val = np.array(flatten_dist_val)
bins = np.linspace(0, MAX_DIST, 20)
fine_bins = np.linspace(0, MAX_DIST, 100)
if model == "LogisticDistance" or model == "LogisticDistanceFixedLambda":
# compute prob as a function of distance for this class
htrue, _ = np.histogram(conn_dist_hist, bins)
hfalse, _ = np.histogram(noconn_dist_hist, bins)
p = htrue.astype(float) / (hfalse + htrue)
ax.plot(bins[:-1], p, c='b', linewidth=3)
if model == "LogisticDistance":
c = ss[(c1, c2)]
print "MAX_DISTANCE=", MAX_DIST, np.max(fine_bins), np.max(bins), c
y = util.logistic(fine_bins, c['mu'], c['lambda'])
y = y * (hps['p_max'] - hps['p_min']) + hps['p_min']
ax.plot(fine_bins, y, c='r', linewidth=2)
ax.text(0, 0.2, r"mu: %3.2f" % c['mu'], fontsize=4)
ax.text(0, 0.6, r"lamb: %3.2f" % c['lambda'], fontsize=4)
ax.axvline(c['mu'], c='k')
elif model == "LogisticDistanceFixedLambda":
c = ss[(c1, c2)]
y = util.logistic(fine_bins, c['mu'], hps['lambda'])
y = y * (c['p_scale'] - hps['p_min']) + hps['p_min']
ax.plot(fine_bins, y, c='r')
ax.text(0, 0.2, r"mu: %3.2f" % c['mu'], fontsize=4)
ax.text(0, 0.6, r"lamb: %3.2f" % hps['lambda'], fontsize=4)
ax.axvline(c['mu'], c='k')
elif model == "ExponentialDistancePoisson":
if len(flatten_dist_val) > 0:
x_jitter = np.random.normal(0, 0.01, len(flatten_dist_val))
y_jitter = np.random.normal(0, 0.05, len(flatten_dist_val))
ax.scatter(flatten_dist_val[:, 0] + x_jitter,
flatten_dist_val[:, 1] + y_jitter,
edgecolor='none',
s=2)
c = ss[(c1, c2)]
mu = c['mu']
rate_scale = c['rate_scale']
y = np.exp(-fine_bins/mu)
y = y * rate_scale
ax.plot(fine_bins, y, c='r')
ax.text(0, 0.2, r"mu: %3.2f" % c['mu'], fontsize=4)
ax.text(0, 0.6, r"rate_scale: %3.2f" % c['rate_scale'], fontsize=4)
ax.set_ylim(-1, 20.0)
ax.axvline(c['mu'], c='k')
elif model == "LogisticDistancePoisson":
if len(flatten_dist_val) > 0:
x_jitter = np.random.normal(0, 0.01, len(flatten_dist_val))
y_jitter = np.random.normal(0, 0.05, len(flatten_dist_val))
ax.scatter(flatten_dist_val[:, 0] + x_jitter,
flatten_dist_val[:, 1] + y_jitter,
edgecolor='none',
s=2)
c = ss[(c1, c2)]
y = util.logistic(fine_bins, c['mu'], hps['lambda'])
y = y * (c['rate_scale'] - hps['rate_min']) + hps['rate_min']
ax.plot(fine_bins, y, c='r')
ax.text(0, 1, r"mu: %3.2f" % c['mu'], fontsize=4)
ax.text(0, 3, r"rate_scale: %3.2f" % c['rate_scale'], fontsize=4)
ax.set_ylim(-1, 20.0)
ax.axvline(c['mu'], c='k')
elif model == "LinearDistance":
print "MAX_DISTANCE=", MAX_DIST, np.max(fine_bins), np.max(bins)
c = ss[(c1, c2)]
y = util.linear_dist(fine_bins, c['p'], c['mu'])
y += hps['p_min']
ax.plot(fine_bins, y, c='r')
ax.set_xlim(0, MAX_DIST)
def derivative_L(zlc):
_zeta, _lam, _c = unpack_zlc(zeta, lam, c,
zlc) # unpack the parameters
Pstar = logistic(dev_zeta_lam(_zeta, _lam, theta))
P = scale_guessing(Pstar, _c)
return DL(theta, r, f, P, _zeta, _lam, _c, Pstar)
def evaluate(model, split, n_samples=None):
"""
Evaluate a trained model.
:param model: the model to evaluate. Can be any made, maf, or real nvp
:param split: string, the data split to evaluate on. Must be 'trn', 'val' or 'tst'
:param n_samples: number of samples to generate from the model, or None for no samples
"""
assert is_data_loaded(), 'Dataset hasn\'t been loaded'
# choose which data split to evaluate on
data_split = getattr(data, split, None)
if data_split is None:
raise ValueError('Invalid data split')
if is_conditional(model):
# calculate log probability
logprobs = model.eval([data_split.y, data_split.x])
print('logprob(x|y) = {0:.2f} +/- {1:.2f}').format(
logprobs.mean(), 2 * logprobs.std() / np.sqrt(data_split.N))
# classify test set
logprobs = np.empty([data_split.N, data.n_labels])
for i in xrange(data.n_labels):
y = np.zeros([data_split.N, data.n_labels])
y[:, i] = 1
logprobs[:, i] = model.eval([y, data_split.x])
predict_label = np.argmax(logprobs, axis=1)
accuracy = (predict_label == data_split.labels).astype(float)
logprobs = scipy.misc.logsumexp(logprobs, axis=1) - np.log(
logprobs.shape[1])
print('logprob(x) = {0:.2f} +/- {1:.2f}'.format(
logprobs.mean(), 2 * logprobs.std() / np.sqrt(data_split.N)))
print('classification accuracy = {0:.2%} +/- {1:.2%}'.format(
accuracy.mean(), 2 * accuracy.std() / np.sqrt(data_split.N)))
# generate data conditioned on label
if n_samples is not None:
for i in xrange(data.n_labels):
# generate samples and sort according to log prob
y = np.zeros(data.n_labels)
y[i] = 1
samples = model.gen(y, n_samples)
lp_samples = model.eval([np.tile(y, [n_samples, 1]), samples])
lp_samples = lp_samples[np.logical_not(np.isnan(lp_samples))]
idx = np.argsort(lp_samples)
samples = samples[idx][::-1]
if data_name == 'mnist':
samples = (util.logistic(samples) -
data.alpha) / (1 - 2 * data.alpha)
elif data_name == 'bsds300':
samples = np.hstack(
[samples, -np.sum(samples, axis=1)[:, np.newaxis]])
elif data_name == 'cifar10':
samples = (util.logistic(samples) -
data.alpha) / (1 - 2 * data.alpha)
D = int(data.n_dims / 3)
r = samples[:, :D]
g = samples[:, D:2 * D]
b = samples[:, 2 * D:]
samples = np.stack([r, g, b], axis=2)
else:
raise ValueError('non-image dataset')
util.disp_imdata(samples, data.image_size, [5, 8])
else:
# calculate average log probability
logprobs = model.eval(data_split.x)
print('logprob(x) = {0:.2f} +/- {1:.2f}'.format(
logprobs.mean(), 2 * logprobs.std() / np.sqrt(data_split.N)))
# generate data
if n_samples is not None:
# generate samples and sort according to log prob
samples = model.gen(n_samples)
lp_samples = model.eval(samples)
lp_samples = lp_samples[np.logical_not(np.isnan(lp_samples))]
idx = np.argsort(lp_samples)
samples = samples[idx][::-1]
if data_name == 'mnist':
samples = (util.logistic(samples) -
data.alpha) / (1 - 2 * data.alpha)
elif data_name == 'bsds300':
samples = np.hstack(
[samples, -np.sum(samples, axis=1)[:, np.newaxis]])
elif data_name == 'cifar10':
samples = (util.logistic(samples) -
data.alpha) / (1 - 2 * data.alpha)
D = int(data.n_dims / 3)
r = samples[:, :D]
g = samples[:, D:2 * D]
b = samples[:, 2 * D:]
samples = np.stack([r, g, b], axis=2)
else:
raise ValueError('non-image dataset')
util.disp_imdata(samples, data.image_size, [5, 8])
plt.show()
def evaluate_logprob(model,
split,
use_image_space=False,
return_avg=True,
batch=2000):
"""
Evaluate a trained model only in terms of log probability.
:param model: the model to evaluate. Can be any made, maf, or real nvp
:param split: string, the data split to evaluate on. Must be 'trn', 'val' or 'tst'
:param use_image_space: bool, whether to report log probability in [0, 1] image space (only for cifar and mnist)
:param return_avg: bool, whether to return average log prob with std error, or all log probs
:param batch: batch size to use for computing log probability
:return: average log probability & standard error, or all log probs
"""
assert is_data_loaded(), 'Dataset hasn\'t been loaded'
# choose which data split to evaluate on
data_split = getattr(data, split, None)
if data_split is None:
raise ValueError('Invalid data split')
if is_conditional(model):
logprobs = np.empty([data_split.N, data.n_labels])
for i in xrange(data.n_labels):
# create labels
y = np.zeros([data_split.N, data.n_labels])
y[:, i] = 1
# process data in batches to make sure they fit in memory
r, l = 0, batch
while r < data_split.N:
logprobs[r:l, i] = model.eval([y[r:l], data_split.x[r:l]])
l += batch
r += batch
logprobs = scipy.misc.logsumexp(logprobs, axis=1) - np.log(
logprobs.shape[1])
else:
logprobs = np.empty(data_split.N)
# process data in batches to make sure they fit in memory
r, l = 0, batch
while r < data_split.N:
logprobs[r:l] = model.eval(data_split.x[r:l])
l += batch
r += batch
if use_image_space:
assert data_name in ['mnist', 'cifar10']
z = util.logistic(data_split.x)
logprobs += data.n_dims * np.log(1 - 2 * data.alpha) - np.sum(
np.log(z) + np.log(1 - z), axis=1)
if return_avg:
avg_logprob = logprobs.mean()
std_err = logprobs.std() / np.sqrt(data_split.N)
return avg_logprob, std_err
else:
return logprobs
def predict(self, x):
return logistic(np.dot(x, self.weights))
def plot_t1t1_params(fig,
conn_and_dist,
assign_vect,
ss,
hps,
MAX_DIST=10,
model="LogisticDistance",
MAX_CLASSES=20):
"""
In the same order that we would plot the latent matrix, plot
the per-parameter properties
hps are per-relation hps
note, tragically, this wants the whole figure
"""
from mpl_toolkits.axes_grid1 import Grid
from matplotlib import pylab
assign_vect = np.array(assign_vect)
canon_assign_vect = util.canonicalize_assignment(assign_vect)
# create the mapping between existing and new
canon_to_old = {}
for i, v in enumerate(canon_assign_vect):
canon_to_old[v] = assign_vect[i]
CLASSES = np.sort(np.unique(canon_assign_vect))
CLASSN = len(CLASSES)
if CLASSN > MAX_CLASSES:
print "WARNING, TOO MANY CLASSES"
CLASSN = MAX_CLASSES
img_grid = Grid(
fig,
111, # similar to subplot(111)
nrows_ncols=(CLASSN, CLASSN),
axes_pad=0.1,
add_all=True,
share_all=True,
label_mode='L',
)
if "istance" not in model:
return
for c1i, c1_canon in enumerate(CLASSES[:MAX_CLASSES]):
for c2i, c2_canon in enumerate(CLASSES[:MAX_CLASSES]):
c1 = canon_to_old[c1_canon]
c2 = canon_to_old[c2_canon]
ax_pos = c1i * CLASSN + c2i
ax = img_grid[ax_pos]
nodes_1 = np.argwhere(assign_vect == c1).flatten()
nodes_2 = np.argwhere(assign_vect == c2).flatten()
conn_dist_hist = []
noconn_dist_hist = []
flatten_dist_val = []
assert len(nodes_1) > 0
assert len(nodes_2) > 0
for n1 in nodes_1:
for n2 in nodes_2:
d = conn_and_dist[n1, n2]['distance']
if conn_and_dist[n1, n2]['link']:
conn_dist_hist.append(d)
else:
noconn_dist_hist.append(d)
flatten_dist_val.append((d, conn_and_dist[n1, n2]['link']))
flatten_dist_val = np.array(flatten_dist_val)
bins = np.linspace(0, MAX_DIST, 20)
fine_bins = np.linspace(0, MAX_DIST, 100)
if model == "LogisticDistance" or model == "LogisticDistanceFixedLambda":
# compute prob as a function of distance for this class
htrue, _ = np.histogram(conn_dist_hist, bins)
hfalse, _ = np.histogram(noconn_dist_hist, bins)
p = htrue.astype(float) / (hfalse + htrue)
ax.plot(bins[:-1], p, c='b', linewidth=3)
if model == "LogisticDistance":
c = ss[(c1, c2)]
print "MAX_DISTANCE=", MAX_DIST, np.max(fine_bins), np.max(
bins), c
y = util.logistic(fine_bins, c['mu'], c['lambda'])
y = y * (hps['p_max'] - hps['p_min']) + hps['p_min']
ax.plot(fine_bins, y, c='r', linewidth=2)
ax.text(0, 0.2, r"mu: %3.2f" % c['mu'], fontsize=4)
ax.text(0, 0.6, r"lamb: %3.2f" % c['lambda'], fontsize=4)
ax.axvline(c['mu'], c='k')
elif model == "LogisticDistanceFixedLambda":
c = ss[(c1, c2)]
y = util.logistic(fine_bins, c['mu'], hps['lambda'])
y = y * (c['p_scale'] - hps['p_min']) + hps['p_min']
ax.plot(fine_bins, y, c='r')
ax.text(0, 0.2, r"mu: %3.2f" % c['mu'], fontsize=4)
ax.text(0, 0.6, r"lamb: %3.2f" % hps['lambda'], fontsize=4)
ax.axvline(c['mu'], c='k')
elif model == "ExponentialDistancePoisson":
if len(flatten_dist_val) > 0:
x_jitter = np.random.normal(0, 0.01, len(flatten_dist_val))
y_jitter = np.random.normal(0, 0.05, len(flatten_dist_val))
ax.scatter(flatten_dist_val[:, 0] + x_jitter,
flatten_dist_val[:, 1] + y_jitter,
edgecolor='none',
s=2)
c = ss[(c1, c2)]
mu = c['mu']
rate_scale = c['rate_scale']
y = np.exp(-fine_bins / mu)
y = y * rate_scale
ax.plot(fine_bins, y, c='r')
ax.text(0, 0.2, r"mu: %3.2f" % c['mu'], fontsize=4)
ax.text(0,
0.6,
r"rate_scale: %3.2f" % c['rate_scale'],
fontsize=4)
ax.set_ylim(-1, 20.0)
ax.axvline(c['mu'], c='k')
elif model == "LogisticDistancePoisson":
if len(flatten_dist_val) > 0:
x_jitter = np.random.normal(0, 0.01, len(flatten_dist_val))
y_jitter = np.random.normal(0, 0.05, len(flatten_dist_val))
ax.scatter(flatten_dist_val[:, 0] + x_jitter,
flatten_dist_val[:, 1] + y_jitter,
edgecolor='none',
s=2)
c = ss[(c1, c2)]
y = util.logistic(fine_bins, c['mu'], hps['lambda'])
y = y * (c['rate_scale'] - hps['rate_min']) + hps['rate_min']
ax.plot(fine_bins, y, c='r')
ax.text(0, 1, r"mu: %3.2f" % c['mu'], fontsize=4)
ax.text(0,
3,
r"rate_scale: %3.2f" % c['rate_scale'],
fontsize=4)
ax.set_ylim(-1, 20.0)
ax.axvline(c['mu'], c='k')
elif model == "LinearDistance":
print "MAX_DISTANCE=", MAX_DIST, np.max(fine_bins), np.max(
bins)
c = ss[(c1, c2)]
y = util.linear_dist(fine_bins, c['p'], c['mu'])
y += hps['p_min']
ax.plot(fine_bins, y, c='r')
ax.set_xlim(0, MAX_DIST)
def mle_3_parameter(theta, r, f, a, b, c, MAX_ITERATIONS=10000):
'''Three parameter logistic ICC model parameter estimation using MLE
From "Item Response Theory: Parameter Estimation Techniques"
Chapter 2.6, pages 48-57 (esp final eqn on p.55)
For comparison,
a = lambda
-a * b = zeta
(structure based on the 2PL function above)
Proof that Qstar = 1 - Pstar:
Q / Qstar = (1 - c) ()
Q = (1 - c) * Qstar
P = c + (1 - c) * Pstar
Q = 1 - P = 1 - c - (1 - c) * Pstar
So, 1 - c - (1 - c) * Pstar = (1 - c) * Qstar
Qstar = ((1 - c) - (1 - c) * Pstar) / (1 - c) = 1 - Pstar'''
theta, r, f = map(np.asanyarray, [theta, r, f]) # ensure these are arrays
p = r / f
for i in range(MAX_ITERATIONS):
print "iteration", i
aa = a * np.ones(f.shape) # array version of a
Pstar = logistic(dev_ab(a, b, theta))
P = np.squeeze(scale_guessing(Pstar, c))
pmP = p - P
iPc = 1 / (P - c)
Q = (1 - P)
Qic = Q / (1 - c)
rat = Pstar / P
thmb = theta - b
#W = Pstar * (1 - Pstar) / (P * (1 - P))
#W[np.where(W<W_CUTOFF)] = np.nan # Delete any dud values
#if np.any(np.nansum(f * W)<0):
# raise ConvergenceError("OUT OF BOUNDS ERROR ITERATION %i" % i)
# LL = np.sum(r * np.log(P)) + np.sum((f - r) * np.log(1 - P)) # Compute the log-likelihood
L1, L2, L3 = np.array([thmb, -aa, iPc
]) * f * pmP * rat # This is right
JLL = np.nansum([L1, L2, L3], axis=-1)
rhs = np.nansum([[L1], [L2], [L3]], axis=-1)
EL11, EL22, EL33 = np.array(
[-P * Q * thmb**2 * rat, -a**2 * P * Q * rat, Qic * iPc]) * f * rat
EL12, EL13, EL23 = np.array(
[a * thmb * P * Q * rat, -thmb * Qic, a * Qic]) * f * rat
# This was wrong, but somehow seemed to work just as well??
#EL11, EL22, EL33 = np.array([-P * Q * thmb ** 2, -a**2 * P * Q, Qic * iPc]) * f * rat**2
#EL12, EL13, EL23 = np.array([a * thmb * P * Q * rat, -thmb * Qic, a * Qic]) * f * rat
mat = JJLL = np.nansum(
[[EL11, EL12, EL13], [EL12, EL22, EL23], [EL13, EL23, EL33]],
axis=-1)
Da, Db, Dc = np.dot(np.linalg.inv(mat), rhs).flatten()
if np.linalg.det(mat) <= DM_CUTOFF:
break
a += Da
b += Db
c += Dc
if abs(a) > MAX_LAM or abs(b) > MAX_B or abs(c) > 1:
raise ConvergenceError("OUT OF BOUNDS ERROR ITERATION %i" % i)
if abs(Da) <= .05 and abs(Db) <= .05 and abs(Dc) <= .05:
break
if i == MAX_ITERATIONS:
print "REACHED MAXIMUM NUMBER OF ITERATIONS"
P = logistic3PL(a, b, c, theta)
chi2 = chiSquared(f, p, P)
return a, b, c, chi2
def update_weights(self, x, y, lr):
grad_log_lik = x.T.dot((y - logistic(x.dot(self.weights))))
self.weights += lr * grad_log_lik
return
def logistic_log_likelihood(weights, x, y):
output_probs = logistic(np.dot(x, weights))
return log_likelihood(y_true=y, y_pred=output_probs)
def logistic_ll_jacobian(weights, x, y):
return x.T.dot((y - logistic(x.dot(weights))))
def __predict_single_class(self, x, class_index):
return logistic(self.__weight[class_index].dot(x) +
self.__bias[class_index])
def __calc_gradient(self, theta, x, y):
y_pred = logistic(x.dot(theta))
_, data_size = x.shape
return ((y_pred - y).dot(x)).transpose() / data_size
def __calc_loss(self, theta, x, y):
y_pred = logistic(x.dot(theta))
return calc_loss(y_pred, y)
def predict_prob(self, x):
return logistic(self.weight.dot(x) + self.bias)
def predict(self, x):
return logistic(self.__weight.dot(x) + self.__bias)