def run(path=resource_manager.Properties.getDefaultDataFold()+"txt"+resource_manager.getSeparator()+"build.txt",sep=' '):
'''
return cluster id
i,j distance
'''
(dist,xxdist,ND,N) = readfile(path, dimensions = 2, sep=sep)
XY, eigs = mds(dist)
(rho,delta,ordrho,dc,nneigh) = rhodelta(dist, xxdist, ND, N, percent = 2.0)
DCplot(dist, XY, ND, rho, delta,ordrho,dc,nneigh,17,0.1)
def recon(D, method):
if method == 'SDP':
return recover_coords(D, method='SDP')
elif method == 'MDS':
return mds(D)
elif method == 'FM':
return MatrixTo3D(D)[0]
else:
print("Unsupported method!")
def run(*args, **kwargs):
'''
return cluster id
'''
file = kwargs.get('fi')
sep = kwargs.get('sep',' ')
########
(dist,xxdist,ND,N) = readfile(file, dimensions = 2, sep=sep)
XY, eigs = mds(dist)
(rho,delta,ordrho,dc,nneigh) = rhodelta(dist, xxdist, ND, N, percent = 2.0)
DCplot(dist, XY, ND, rho, delta,ordrho,dc,nneigh,17,0.1,31)
def callback(iters, current, env):
print 'Callback: ', iters
# get all traces
env.updategui = False
traces = env.evaluate_policy(current)
env.updategui = True
pickle.dump(traces, open('traces{0}.pck'.format(iters), 'w'),
pickle.HIGHEST_PROTOCOL)
# measure task performance
avg_reward = 0
for t in traces:
avg_reward += sum([i[2] for i in t])
avg_reward = avg_reward / float(len(traces))
print 'Avg reward: ', avg_reward
# find current embedding
ematrix = np.zeros((512, 512))
for (i, t) in enumerate(traces):
for (j, s) in enumerate(traces):
ematrix[i,
j] = edit_distance_vc([e[1] for e in t], [l[1] for l in s],
(1.0, 1.0, 1.2))
pickle.dump(ematrix, open('ematrix{0}.pck'.format(iters), 'w'),
pickle.HIGHEST_PROTOCOL)
y, s, adj = isomap(ematrix)
if len(y) < 512:
# fallback to mds if more than 1 connected component
print "More than 1 CC - Falling back to MDS"
y, s = mds(ematrix)
adj = None
# plot stuff later because of pylab / pygame incompat on mac
# save embedding image - multiple formats?
#scatter(y[:,0],y[:,1], filename='scatter{0}.pdf'.format(iters))
# save scree plot
#plot(s[:10], filename='scree{0}.pdf'.format(iters))
# procrustes error
gt = env.coords_array()
err = procrustes(gt, y)
print "Procrustes ", err
pickle.dump((iters, err, gt, avg_reward, current, y, s, adj),
open('misc{0}.pck'.format(iters), 'w'),
pickle.HIGHEST_PROTOCOL)
env.save('iter{0}.png'.format(iters))
def isomap(d, fnc='k', size=7, dimensions=2):
""" Compute isomap instead of mds. Currently neighborhoods of type k are supported. """
if not len(d.shape) == 2:
raise ValueError("d must be a square matrix")
# the put operation is destructive to d
ld = cluster_graph(d, fnc=fnc, size=size, graph='distance')
adj = cluster_graph(d, fnc=fnc, size=size, graph='adjacency')
# shortest paths will find connected components
tmp = np.less(ld, fpconst.PosInf) # 0-1
groups = list_components(tmp)
# now do classical mds on largest connected component
groups.sort(lambda x, y: cmp(len(x), len(y)), reverse=True)
dg = slice_matrix(ld, groups[0], groups[0])
Y, eigs = mds(dg, dimensions)
return (Y, eigs, adj)
def isomap(d, fnc = 'k', size = 7, dimensions = 2):
""" Compute isomap instead of mds. Currently neighborhoods of type k are supported. """
if not len(d.shape) == 2:
raise ValueError, "d must be a square matrix"
# the put operation is destructive to d
ld = cluster_graph(d, fnc = fnc, size = size, graph = 'distance')
adj = cluster_graph(d, fnc = fnc, size = size, graph = 'adjacency')
# shortest paths will find connected components
tmp = np.less(ld, fpconst.PosInf) # 0-1
groups = list_components(tmp)
# now do classical mds on largest connected component
groups.sort(lambda x,y: cmp(len(x),len(y)), reverse=True)
dg = slice_matrix(ld, groups[0], groups[0])
Y,eigs = mds(dg, dimensions)
return (Y, eigs, adj)
for k in range(13):
# traces = pickle.load(open('traces{0}.pck'.format(k)))
# # find current embedding
# ematrix = np.zeros((512,512))
# for (i,t) in enumerate(traces):
# for (j,s) in enumerate(traces):
# #ematrix[i,j] = edit_distance_vc([e[1] for e in t], [l[1] for l in s], (1.0, 1.0, 1.5))
# ematrix[i,j] = non_dtw_distance([e[1] for e in t], [l[1] for l in s], default = 8, costf=adist)
# pickle.dump(ematrix,open('ematrix_revised{0}'.format(k),'w'),pickle.HIGHEST_PROTOCOL)
ematrix = pickle.load(open('ematrix_revised{0}'.format(k)))
y,s = mds(ematrix)
pylab.clf()
pylab.title("Iteration {0}".format(k))
pylab.scatter(y[:,0],y[:,1],c=colors)
pylab.savefig("embed_{0}".format(k))
if False:
traces = pickle.load(open('traces13.pck'))
# find current embedding
ematrix = np.zeros((512,512))
for (i,t) in enumerate(traces):
for (j,s) in enumerate(traces):
#ematrix[i,j] = edit_distance_vc([e[1] for e in t], [l[1] for l in s], (1.0, 1.0, 1.5))
ematrix[i,j] = non_dtw_distance([e[1] for e in t], [l[1] for l in s], default = 8, costf=adist)
y,s = mds(ematrix)#isomap(ematrix)
pylab.scatter(y[:,0],y[:,1])
# svd
tfros, tvars, tstr, mbs = svd(Ro, Go, Bo, index)
Fros[0, :] = transpose(tfros)
var[0, :] = transpose(tvars)
Stress[0, :] = transpose(tstr)
storages[0, :] = transpose(mbs)
# pca
tfros, tvars, tstr, mbs = pca(Ro, Go, Bo, index)
Fros[1, :] = transpose(tfros)
var[1, :] = transpose(tvars)
Stress[1, :] = transpose(tstr)
storages[1, :] = transpose(mbs)
# mds
tfros, tvars, tstr, mbs = mds(Ro, Go, Bo, index)
Fros[2, :] = transpose(tfros)
var[2, :] = transpose(tvars)
Stress[2, :] = transpose(tstr)
storages[2, :] = transpose(mbs)
storages = storages / storage_o
# plot images
fig = plt.figure()
# plot frobenius norm for the three methods
a = fig.add_subplot(221)
plt.plot(log10(index[0]), Fros[0, :], color='r', linewidth=2.0, label='svd')
plt.plot(log10(index[0]), Fros[1, :], color='g', label='pca')
plt.plot(log10(index[0]), Fros[2, :], color='b', label='mds')
plt.xlabel('k log10 scale')
xlabel='Unemployment rate (%)',
ylabel='murders per year per 1,000,000 inhabitants',
title='Murder rates in correlaction with unemployment \n (Centered)',
legend_labels=(''),
arrows=arrows,
equal_axis=False,
tickfrequencyone=False)
weed_crop_train = np.loadtxt('IDSWeedCropTrain.csv', delimiter=',')
weed_crop_train = weed_crop_train[:, :-1]
weed_crop_test = np.loadtxt('IDSWeedCropTest.csv', delimiter=',')
eigenvalues_p, eigenvectors_p, mean_p = pca(weed_crop_train)
datamatrix = mds(weed_crop_train, 2, show_pc_plots=True)
plotify.scatter_plot(x_list=[datamatrix[0, :]],
y_list=[datamatrix[1, :]],
tickfrequencyone=False,
xlabel='PC 1',
ylabel='PC 2',
title='First 2 PCs of the Pesticide dataset')
datamatrix = mds(weed_crop_train, 3, show_pc_plots=False)
plotify.scatter3d(x=datamatrix[0, :],
y=datamatrix[1, :],
z=datamatrix[2, :],
xlabel='PC 1',
ylabel='PC 2',
for row in cfd_mds_data[:-1]:
cols = row.split('\t')
f1 = int(cols[0])
f2 = int(cols[1])
imageLocations[f1] = cols[2]
imageLocations[f2] = cols[3]
similarity = float(cols[4])
difference_matrix[f1][f2] = (comparison_count[f1][f2] * difference_matrix[f1][f2] + similarity) / (difference_matrix[f1][f2] + 1)
for y, row in enumerate(difference_matrix):
for x, cell in enumerate(row):
if cell == 0:
if x != y:
difference_matrix[y][x] = average_similarity
else:
difference_matrix[y][x] = 7
difference_matrix[y][x] -= 7
difference_matrix[y][x] *= -1
positions2d, eigs2d = mds.mds(difference_matrix, 2)
positions1d, eigs1d = mds.mds(difference_matrix, 1)
locations2d = file('2dlocations.txt', 'w+')
locations1d = file('1dlocations.txt', 'w+')
for index, position in enumerate(positions2d):
locations2d.write('"{0}"\t{1}\t{2}\n'.format(imageLocations[index], position[0], position[1]))
for index, position in enumerate(positions1d):
locations1d.write('"{0}"\t{1}\n'.format(imageLocations[index], position[0]))
pool = Pool(6)
results = []
traces[272] = [(272, 0, 0.0, 272)] # fix for empty trace
for (i, t) in enumerate(traces):
for (j, s) in enumerate(traces):
r = pool.apply_async(dtw_apply, (i, j, t, s))
results.append(r)
ematrix = np.zeros((512, 512))
for r in results:
(i, j, d) = r.get()
ematrix[i, j] = d
# ematrix[i,j] = dtw_distance([e[1] for e in t], [l[1] for l in s], costf=cost_func)
np.save("test_matrix.npy", ematrix)
y, s = mds(ematrix)
# pylab.scatter(y[:,0],y[:,1])
# pylab.show()
if False:
traces = pickle.load(open("perf_pca_traces.pck"))
from dtw import edit_distance_vc
ematrix = np.zeros((512, 512))
for (i, t) in enumerate(traces):
for (j, s) in enumerate(traces):
ematrix[i, j] = edit_distance_vc([e[1] for e in t], [l[1] for l in s], (1.0, 1.0, 1.2))
np.save("ematrix5.npy", ematrix)
for k in range(13):
# traces = pickle.load(open('traces{0}.pck'.format(k)))
# # find current embedding
# ematrix = np.zeros((512,512))
# for (i,t) in enumerate(traces):
# for (j,s) in enumerate(traces):
# #ematrix[i,j] = edit_distance_vc([e[1] for e in t], [l[1] for l in s], (1.0, 1.0, 1.5))
# ematrix[i,j] = non_dtw_distance([e[1] for e in t], [l[1] for l in s], default = 8, costf=adist)
# pickle.dump(ematrix,open('ematrix_revised{0}'.format(k),'w'),pickle.HIGHEST_PROTOCOL)
ematrix = pickle.load(open('ematrix_revised{0}'.format(k)))
y, s = mds(ematrix)
pylab.clf()
pylab.title("Iteration {0}".format(k))
pylab.scatter(y[:, 0], y[:, 1], c=colors)
pylab.savefig("embed_{0}".format(k))
if False:
traces = pickle.load(open('traces13.pck'))
# find current embedding
ematrix = np.zeros((512, 512))
for (i, t) in enumerate(traces):
for (j, s) in enumerate(traces):
#ematrix[i,j] = edit_distance_vc([e[1] for e in t], [l[1] for l in s], (1.0, 1.0, 1.5))
ematrix[i, j] = non_dtw_distance([e[1] for e in t],
[l[1] for l in s],
default=8,
# svd
tfros,tvars,tstr,mbs = svd(Ro,Go,Bo,index)
Fros[0,:] = transpose(tfros)
var[0,:] = transpose(tvars)
Stress[0,:] = transpose(tstr)
storages[0,:] = transpose(mbs)
# pca
tfros,tvars,tstr,mbs = pca(Ro,Go,Bo,index)
Fros[1,:] = transpose(tfros)
var[1,:] = transpose(tvars)
Stress[1,:] = transpose(tstr)
storages[1,:] = transpose(mbs)
# mds
tfros,tvars,tstr,mbs = mds(Ro,Go,Bo,index)
Fros[2,:] = transpose(tfros)
var[2,:] = transpose(tvars)
Stress[2,:] = transpose(tstr)
storages[2,:] = transpose(mbs)
storages = storages/storage_o
# plot images
fig = plt.figure()
# plot frobenius norm for the three methods
a = fig.add_subplot(221)
plt.plot(log10(index[0]),Fros[0,:],color='r',linewidth=2.0,label='svd')
plt.plot(log10(index[0]),Fros[1,:],color='g',label='pca')
plt.plot(log10(index[0]),Fros[2,:],color='b',label='mds')
plt.xlabel('k log10 scale')
action_traces.append([j[1] for j in v.single_episode(policy, start=i)])
print action_traces
def costf(a, b):
return np.abs(a - b)
n = len(action_traces)
ematrix = np.zeros((n, n))
for (i, t) in enumerate(action_traces):
for (j, s) in enumerate(action_traces):
ematrix[i, j] = non_dtw_distance(t, s, default=0, costf=costf)
y, s = mds(ematrix)
pylab.clf()
pylab.title("Sensorimotor Distances")
colors = ['red', 'orange', 'green', 'blue']
pts = []
for i in range(4):
lbl = ""
if i == 0:
lbl = "Farthest Object"
elif i == 3:
lbl = "Closest Object"
else:
lbl = "None"
print lbl
pts.append(pylab.scatter([y[i, 0]], [y[i, 1]], c=colors[i], label=lbl))
# pylab.gca().set_xticklabels([])
# gw.set_arrows(pi)
# gw.background()
# dump weights in case we need them in the future
pickle.dump(weights0, open("exp_comb/weights.pck", "w"), pickle.HIGHEST_PROTOCOL)
# generate traces for sensorimotor embedding
traces = gw.evaluate_policy(w0)
pickle.dump(traces, open("exp_comb/comb_traces.pck", "w"), pickle.HIGHEST_PROTOCOL)
ematrix = np.zeros((49, 49))
for (i, t) in enumerate(traces):
for (j, s) in enumerate(traces):
ematrix[i, j] = edit_distance([e[1] for e in t], [l[1] for l in s])
y, s = mds(ematrix)
pylab.clf()
pylab.scatter(y[:, 0], y[:, 1], c=colors)
save_many('exp_comb/se')
#
# MVU
#
N = gw.neighbors([0, 2, 4, 6])
# base colors on true points
pts = np.array(gw.states.values())
colors = create_norm_colors(pts)
action_traces = []
for i in [91,93,95,97]:
action_traces.append([j[1] for j in v.single_episode(policy, start=i)])
print action_traces
def costf(a,b):
return np.abs(a-b)
n = len(action_traces)
ematrix = np.zeros((n, n))
for (i, t) in enumerate(action_traces):
for (j, s) in enumerate(action_traces):
ematrix[i, j] = non_dtw_distance(t,s,default=0, costf=costf)
y,s = mds(ematrix)
pylab.clf()
pylab.title("Sensorimotor Distances")
colors = ['red','orange','green','blue']
pts = []
for i in range(4):
lbl = ""
if i == 0:
lbl = "Farthest Object"
elif i == 3:
lbl = "Closest Object"
else:
lbl = "None"
print lbl
pts.append(pylab.scatter([y[i,0]],[y[i,1]],c=colors[i],label=lbl))
# pylab.gca().set_xticklabels([])
from proximityMatrix import proximityMatrix
proxMat = proximityMatrix(clf, X_train)
print("proxMat: \n", proxMat)
from sklearn.manifold import TSNE
embeddedModel = TSNE(n_components=2).fit_transform(proxMat)
print(embeddedModel.shape)
embeddedModel_ = TSNE(n_components=6).fit_transform(proxMat)
print(embeddedModel_.shape)
from mds import mds, square_points
Y, eigs = mds(proxMat)
pylab.figure(1)
pylab.plot(Y[:,0],Y[:,1],'.')
pylab.figure(2)
pylab.plot(points[:,0], points[:,1], '.')
pylab.show()
import numpy as np
from mds import mds
import matplotlib.pyplot as plt
import matplotlib.cm as cm
if __name__ == "__main__":
Cities = np.array([
'BeiJing', 'ShangHai', 'HeZe', 'GuangZhou', 'DaTong', 'XiAn', 'HerBin',
'NanJing'
])
D = np.matrix(
'0 2.16 4.68 3.33 5.85 2.08 1.91 2.08;0 0 12.95 2.41 9.33 2.58 2.91 1.08;0 0 0 17.91 11.76 9.56 21 8;0 0 0 0 36 2.58 4.16 2.25;0 0 0 0 0 16.26 24 16.88;0 0 0 0 0 0 2.91 2;0 0 0 0 0 0 0 2.83 ;0 0 0 0 0 0 0 0'
)
X = mds((D + np.transpose(D)) / 2)
ArrayX = np.array(X)
colors = cm.rainbow(np.linspace(0, 1, len(ArrayX)))
for x, c, name in zip(ArrayX, colors, Cities):
plt.scatter(x[0], x[1], label=name, color=c)
plt.legend()
plt.title('City locations under MDS')
plt.show()
# dump weights in case we need them in the future
pickle.dump(weights0, open("exp_comb/weights.pck", "w"),
pickle.HIGHEST_PROTOCOL)
# generate traces for sensorimotor embedding
traces = gw.evaluate_policy(w0)
pickle.dump(traces, open("exp_comb/comb_traces.pck", "w"),
pickle.HIGHEST_PROTOCOL)
ematrix = np.zeros((49, 49))
for (i, t) in enumerate(traces):
for (j, s) in enumerate(traces):
ematrix[i, j] = edit_distance([e[1] for e in t], [l[1] for l in s])
y, s = mds(ematrix)
pylab.clf()
pylab.scatter(y[:, 0], y[:, 1], c=colors)
save_many('exp_comb/se')
#
# MVU
#
N = gw.neighbors([0, 2, 4, 6])
# base colors on true points
pts = np.array(gw.states.values())
colors = create_norm_colors(pts)
pool = Pool(6)
results = []
traces[272] = [(272, 0, 0.0, 272)] # fix for empty trace
for (i, t) in enumerate(traces):
for (j, s) in enumerate(traces):
r = pool.apply_async(dtw_apply, (i, j, t, s))
results.append(r)
ematrix = np.zeros((512, 512))
for r in results:
(i, j, d) = r.get()
ematrix[i, j] = d
#ematrix[i,j] = dtw_distance([e[1] for e in t], [l[1] for l in s], costf=cost_func)
np.save('test_matrix.npy', ematrix)
y, s = mds(ematrix)
#pylab.scatter(y[:,0],y[:,1])
#pylab.show()
if False:
traces = pickle.load(open("perf_pca_traces.pck"))
from dtw import edit_distance_vc
ematrix = np.zeros((512, 512))
for (i, t) in enumerate(traces):
for (j, s) in enumerate(traces):
ematrix[i,
j] = edit_distance_vc([e[1] for e in t], [l[1] for l in s],
(1.0, 1.0, 1.2))
np.save('ematrix5.npy', ematrix)