def _plot2D(self, embeddings):
args = get_args()
fig = plt.figure()
ax = fig.add_subplot(111)
colors = cm.Spectral(np.linspace(0, 1, self.__num_classes))
xx = embeddings[:, 0]
yy = embeddings[:, 1]
if args.with_images == True:
for i, d in enumerate(zip(xx, yy)):
x, y = d
im = OffsetImage(self.__x_sample[i], zoom=0.1, cmap='gray')
ab = AnnotationBbox(im, (x, y), xycoords='data', frameon=False)
ax.add_artist(ab)
ax.update_datalim(np.column_stack([xx, yy]))
ax.autoscale()
for i in range(self.__num_classes):
ax.scatter(xx[self.__y_sample == i],
yy[self.__y_sample == i],
color=colors[i],
label=self.__labels[i],
s=60)
ax.xaxis.set_major_formatter(NullFormatter())
ax.yaxis.set_major_formatter(NullFormatter())
save_name = self.__save_name(2)
plt.title(save_name)
plt.axis('tight')
plt.grid(True)
plt.legend(loc='best', scatterpoints=1, fontsize=5)
if self._args.plot_dir:
plt.savefig(self._args.plot_dir + save_name, dpi=900)
print 'fig\'s been saved!'
def _plot3D(self, embeddings):
if self._args.with_images:
sys.exit("Cannot plot images with 3D plots.")
fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
colors = cm.Spectral(np.linspace(0, 1, self.__num_classes))
xx = embeddings[:, 0]
yy = embeddings[:, 1]
zz = embeddings[:, 2]
for i in range(self.__num_classes):
ax.scatter(xx[self.__y_sample == i],
yy[self.__y_sample == i],
zz[self.__y_sample == i],
color=colors[i],
label=self.__labels,
s=15)
ax.xaxis.set_major_formatter(NullFormatter())
ax.yaxis.set_major_formatter(NullFormatter())
ax.zaxis.set_major_formatter(NullFormatter())
plt.axis('tight')
plt.legend(loc='best', scatterpoints=1, fontsize=5)
save_name = self.__save_name(3)
plt.grid(True)
if self._args.plot_dir:
plt.savefig(self._args.plot_dir + save_name, dpi=600)
def Tsne_visual(Train, y, filename):
mytargets = list(range(0,10))
XX_train, yy_train = Train, y
num_classes = len(np.unique(yy_train))
labels = np.arange(num_classes)
num_samples_to_plot = 50000
import random
idx = [random.randrange(XX_train.shape[0]) for i in xrange(num_samples_to_plot)]
X_train, y = XX_train[idx,:], yy_train[idx] # lets subsample a bit for a first impression
transformer = TSNE(n_components = 2, perplexity=40, verbose=2)
X_transformed = transformer.fit_transform(X_train)
# Plotting
fig = plt.figure()
ax = fig.add_subplot(111)
colors = cm.Spectral(np.linspace(0, 1,10))
xx = X_transformed [:, 0]
yy = X_transformed [:, 1]
print("xx", xx.shape, "yy", yy.shape)
print("y_train", max(y) )
# plot the 3D data points
for i in range(num_classes):
ax.scatter(xx[y==i], yy[y==i], color=colors[i], label=str(i), s=10)
ax.xaxis.set_major_formatter(NullFormatter())
ax.yaxis.set_major_formatter(NullFormatter())
plt.axis('tight')
plt.legend(loc='best', scatterpoints=1, fontsize=5)
plt.savefig(filename+'.pdf', format='pdf', dpi=600)
def plot_2d(X, y=np.array([0])):
# Plot data
plt.figure(figsize=(10,10))
unique_labels = set(y)
colors = [cm.Spectral(each) for each in np.linspace(0, 1, len(unique_labels))]
for k, col in zip(unique_labels, colors):
class_member_mask = (y == k)
xy = X[class_member_mask]
plt.plot(xy[:, 0], xy[:, 1], 'o', markerfacecolor=tuple(col),
markeredgecolor='k', markersize=6, label='{}'.format(k), alpha=0.1)
plt.legend(loc='best')
plt.title('Data')
def plot_tsne(embeddings, labels, logdir, step):
#tsne = TSNE(perplexity=30, n_components=2, init='pca', n_iter=3000)
#tsne = TSNE(n_components=2, init='pca', verbose=2)
tsne = TSNE(n_components=2)
embeddings = tsne.fit_transform(embeddings)
num_classes = len(np.unique(labels))
fig = plt.figure()
#fig, ax = plt.subplots(1, 1)
#subname = datetime.strftime(datetime.now(), '%Y%m%d-%H%M%S')
ax = fig.add_subplot(111)
colors = cm.Spectral(np.linspace(0, 1, num_classes))
classes = np.arange(num_classes)
xx = embeddings[:, 0]
yy = embeddings[:, 1]
# plot the 2D data points
labels = np.array(labels)
cmarker = [
'.', 'o', ',', 'v', '^', '<', '>', '1', '2', '3', '4', '8', 's', 'p',
'*', 'h', '+', 'd'
]
for i in range(num_classes):
ax.scatter(xx[labels == i],
yy[labels == i],
color=colors[i],
label=classes[i],
s=10,
marker=cmarker[i % len(cmarker)])
# index = list(np.where(labels == i))[0]
# for k in range(len(index)):
# ccmarker = ['o', '+']
# ccolors = ['r','b']
# ax.scatter(xx[index[k]], yy[index[k]], color=ccolors[0 if k < len(index)//2 else 1], label=classes[i], s=10,
# marker=cmarker[i % len(cmarker)])
ax.xaxis.set_major_formatter(NullFormatter())
ax.yaxis.set_major_formatter(NullFormatter())
plt.axis('tight')
plt.legend(loc='best', scatterpoints=1, fontsize=5)
#plt.savefig(os.path.join(logdir, "tnse_"+subname+".png"))
plt.minorticks_on()
plt.savefig(os.path.join(logdir, "tnse_" + str(step) + ".png"),
format='png')
def save_latent_vis(encoder):
means, log_vars = encoder.predict(x_train, batch_size=100)
epsilon = np.random.normal(size=(means.shape[0], latent_dim))
z = means + np.exp(log_vars / 2) * epsilon
fig = plt.figure()
ax = fig.add_subplot(111)
colors = cm.Spectral(np.linspace(0, 1, num_classes))
xx = z[:,0]
yy = z[:,1]
labels = range(num_classes)
# plot the 2D data points
for i in range(num_classes):
ax.scatter(xx[y_train == i], yy[y_train == i], color=colors[i], label=labels[i], s=5)
plt.axis('tight')
plt.savefig('Images/toydset_vae_latent_1.png')
def save_latent_vis(encoder):
z = encoder.predict(x_train)
fig = plt.figure()
ax = fig.add_subplot(111)
colors = cm.Spectral(np.linspace(0, 1, num_classes))
xx = z[:, 0]
yy = z[:, 1]
labels = range(num_classes)
# plot the 2D data points
for i in range(num_classes):
ax.scatter(xx[y_train == i],
yy[y_train == i],
color=colors[i],
label=labels[i],
s=5)
plt.axis('tight')
plt.savefig('Images/toydset_latent_regressor_latent.png')
def plot_decision_boundary(solver, X, y, h=0.3, scale=1., title='decision boundary'):
plt.figure(figsize=(10,10))
unique_labels = set(y)
colors = {l : cm.Spectral(each) for l , each in zip(unique_labels, np.linspace(0, 1, len(unique_labels)))}
pan = 1.0
x_min = X[:, 0].min()
x_min = x_min - pan if x_min <= 0.0 else x_min + pan
x_max = X[:, 0].max() + .5
x_max = x_max - pan if x_max <= 0.0 else x_max + pan
y_min = X[:, 1].min() - .5
y_min = y_min - pan if y_min <= 0.0 else y_min + pan
y_max = X[:, 1].max() + .5
y_max = y_max - pan if y_max <= 0.0 else y_max + pan
xx, yy = np.meshgrid(
np.arange(x_min, x_max, h),
np.arange(y_min, y_max, h)
)
Z = np.array([solver.action(i) for i in np.c_[xx.ravel(), yy.ravel()]/scale])
# Put the result into a color plot.
Z = Z.reshape(xx.shape)
plt.pcolormesh(xx, yy, Z, cmap=cm.Spectral, alpha=.40, shading= 'gouraud')
# Add the training points to the plot.
for k, col in colors.items():
class_member_mask = (y == k)
xy = X[class_member_mask]
plt.plot(xy[:, 0], xy[:, 1], 'o', markerfacecolor=tuple(col),
markeredgecolor='k', markersize=6, label='{}'.format(k))
plt.xlim(xx.min(), xx.max())
plt.ylim(yy.min(), yy.max())
plt.legend(loc='best')
plt.title(title)
plt.show()
def plot_sentence_embedding(gold_embedding, noisy_embedding, perplexity=30):
#
# pdb.set_trace()
id2lab = {0: "normed", 1: "ugc"}
X = np.concatenate((gold_embedding, noisy_embedding), axis=0)
y = np.array([0 for _ in range(gold_embedding.shape[0])] +
[1 for _ in range(noisy_embedding.shape[0])],
dtype=np.int)
embeddings = TSNE(n_components=2,
init='pca',
verbose=2,
perplexity=perplexity,
n_iter=500).fit_transform(X)
xx = embeddings[:, 0]
yy = embeddings[:, 1]
fig = plt.figure()
ax = fig.add_subplot(111)
num_classes = 2
colors = cm.Spectral(np.linspace(0, 1, 5))
labels = np.arange(num_classes)
for i in range(num_classes):
ax.scatter(xx[y == i],
yy[y == i],
color=colors[i],
label=id2lab[i],
s=12)
ax.xaxis.set_major_formatter(NullFormatter())
ax.yaxis.set_major_formatter(NullFormatter())
plt.axis('tight')
plt.legend(loc='best', scatterpoints=1, fontsize=10)
# plt.savefig(output_folder + str(layer_id) +".pdf", format='pdf', dpi=600)
plt.show()
def tsne_visual(features, labels, episode, n_components=2):
"""Visualize high-dimensional data in the feature space with t-distributed Stochastic Neighbor Embedding.
Plotting part is from https://github.com/kevinzakka/tsne-viz/blob/master/main.py
Args:
features: features of all data points. shape (n_samples, n_features)
labels: labels corresponding to all data points. shape (n_samples)
episode: current testing episode
n_components: Dimension of the embedded space.
Returns:
feat_embedded: Embedding of the training data in low-dimensional space. shape (n_samples, n_components)
"""
feat_embedded = TSNE(n_components=n_components).fit_transform(features)
classes = np.unique(labels)
fig = plt.figure()
ax = fig.add_subplot(111)
colors = cm.Spectral(np.linspace(0, 1, len(classes)))
xx = feat_embedded[:, 0]
yy = feat_embedded[:, 1]
#plot the images
for i, class_i in enumerate(classes.tolist()):
ax.scatter(xx[labels == class_i],
yy[labels == class_i],
color=colors[i],
label=str(class_i),
s=10)
ax.xaxis.set_major_formatter(NullFormatter())
ax.yaxis.set_major_formatter(NullFormatter())
plt.axis('tight')
plt.legend(loc='best', scatterpoints=1, fontsize=5)
plt.savefig('./tsne_figure/tsne_' + str(episode) + '.png', dpi=500)
def t_sne(latent_vecs,
final_label,
out_dir,
label_names=['Source Data', 'Target Data']):
num_classes = len(label_names)
fname = "tsne_" + str(time.time()) + '.png'
fig = plt.figure()
ax = fig.add_subplot(111)
colors = cm.Spectral(np.linspace(0, 1, 20))
embeddings = TSNE(n_components=2,
random_state=0).fit_transform(latent_vecs)
xx = embeddings[:, 0]
yy = embeddings[:, 1]
# plot the 2D data points
for i in range(num_classes):
ax.scatter(xx[final_label == i],
yy[final_label == i],
color=colors[i * 3 + 2],
label=label_names[i],
s=20)
ax.xaxis.set_major_formatter(NullFormatter())
ax.set_xticks([])
ax.yaxis.set_major_formatter(NullFormatter())
ax.set_yticks([])
plt.axis('tight')
# plt.xticks([])
# plt.yticks([])
plt.axis('off')
plt.legend(loc='best', scatterpoints=1, fontsize=10)
plt.savefig(out_dir + '/' + fname,
format='png',
dpi=600,
pad_inches=0,
bbox_inches='tight')
def combine_FIDs_report(inFIDs,
outFID,
hdr,
ncha=2,
ppmlim=(0.0, 6.0),
method='not specified',
html=None):
""" Take list of FIDs that are passed to combine and output
If uncombined data it will display ncha channels (default 2).
"""
from fsl_mrs.core import MRS
import plotly.graph_objects as go
from fsl_mrs.utils.preproc.reporting import plotStyles, plotAxesStyle
from matplotlib.pyplot import cm
toMRSobj = lambda fid: MRS(FID=fid, header=hdr)
# Assemble data to plot
toPlotIn = []
colourVecIn = []
legendIn = []
if isinstance(inFIDs, list):
for idx, fid in enumerate(inFIDs):
if inFIDs[0].ndim > 1:
toPlotIn.extend([toMRSobj(f) for f in fid[:, :ncha].T])
colourVecIn.extend([idx / len(inFIDs)] * ncha)
legendIn.extend(
[f'FID #{idx}: CHA #{jdx}' for jdx in range(ncha)])
else:
toPlotIn.append(toMRSobj(fid))
colourVecIn.append(idx / len(inFIDs))
legendIn.append(f'FID #{idx}')
elif inFIDs.ndim > 1:
toPlotIn.extend([toMRSobj(f) for f in inFIDs[:, :ncha].T])
colourVecIn.extend([float(jdx) / ncha for jdx in range(ncha)])
legendIn.extend([f'FID #0: CHA #{jdx}' for jdx in range(ncha)])
toPlotOut = []
legendOut = []
if outFID.ndim > 1:
toPlotOut.extend([toMRSobj(f) for f in outFID[:, :ncha].T])
legendOut.extend([f'Combined: CHA #{jdx}' for jdx in range(ncha)])
else:
toPlotOut.append(toMRSobj(outFID))
legendOut.append('Combined')
def addline(fig, mrs, lim, name, linestyle):
trace = go.Scatter(x=mrs.getAxes(ppmlim=lim),
y=np.real(mrs.getSpectrum(ppmlim=lim)),
mode='lines',
name=name,
line=linestyle)
return fig.add_trace(trace)
lines, colors, _ = plotStyles()
colors = cm.Spectral(np.array(colourVecIn).ravel())
fig = go.Figure()
for idx, fid in enumerate(toPlotIn):
cval = np.round(255 * colors[idx, :])
linetmp = {'color': f'rgb({cval[0]},{cval[1]},{cval[2]})', 'width': 1}
fig = addline(fig, fid, ppmlim, legendIn[idx], linetmp)
for idx, fid in enumerate(toPlotOut):
fig = addline(fig, fid, ppmlim, legendOut[idx], lines['blk'])
plotAxesStyle(fig, ppmlim, 'Combined')
# Generate report
if html is not None:
from plotly.offline import plot
from fsl_mrs.utils.preproc.reporting import figgroup, singleReport
from datetime import datetime
import os.path as op
if op.isdir(html):
filename = 'report_' + datetime.now().strftime(
"%Y%m%d_%H%M%S%f")[:-3] + '.html'
htmlfile = op.join(html, filename)
elif op.isdir(op.dirname(html)) and op.splitext(html)[1] == '.html':
htmlfile = html
else:
raise ValueError('Report html path must be file or directory. ')
opName = 'Combination'
timestr = datetime.now().strftime("%H:%M:%S")
datestr = datetime.now().strftime("%d/%m/%Y")
headerinfo = 'Report for fsl_mrs.utils.preproc.combine.combine_FIDs.\n'\
+ f'Generated at {timestr} on {datestr}.'
# Figures
div = plot(fig, output_type='div', include_plotlyjs='cdn')
figurelist = [
figgroup(fig=div,
name='',
foretext=f'Combination of spectra. Method = {method}',
afttext=f'')
]
singleReport(htmlfile, opName, headerinfo, figurelist)
return fig
else:
return fig
def tsne(x, y, y_pred):
X_train = np.asarray(x).astype('float64')
# shuffle dataset
if 1 == 3:
p = np.random.permutation(len(X_train))
X_train = X_train[p]
y_train = y[p]
num_classes = len(np.unique(y))
labels = np.arange(num_classes)
# restrict to a sample because slow
mask = np.arange(5000)
X_sample = x[mask].squeeze()
y_sample = y[mask]
y_sample_pred = y_pred[mask]
if 1 == 1:
print("X_sample: {}".format(X_sample.shape))
print("y_sample: {}".format(y_sample.shape))
print("y_pred: {}".format(y_sample_pred.shape))
# flatten images to (N, D) for feeding to t-SNE
X_sample_flat = np.reshape(X_sample, [X_sample.shape[0], -1])
# compute tsne embeddings
embeddings = TSNE(n_components=2, init='pca',
verbose=2).fit_transform(X_sample_flat)
# dump
#pickle.dump(embeddings, open(config.data_dir + file_name, "wb"))
if 1 == 1:
fig = plt.figure()
ax = fig.add_subplot(111)
colors = cm.Spectral(np.linspace(0, 1, num_classes))
xx = embeddings[:, 0]
yy = embeddings[:, 1]
# plot the images
if 1 == 2:
for i, (x, y) in enumerate(zip(xx, yy)):
im = OffsetImage(X_sample[i], zoom=0.1, cmap='gray')
ab = AnnotationBbox(im, (x, y), xycoords='data', frameon=False)
ax.add_artist(ab)
ax.update_datalim(np.column_stack([xx, yy]))
ax.autoscale()
# plot the 2D data points
for i in range(num_classes):
ax.scatter(xx[y_sample == i],
yy[y_sample == i],
color=colors[i],
label=labels[i],
s=10)
ax.xaxis.set_major_formatter(NullFormatter())
ax.yaxis.set_major_formatter(NullFormatter())
plt.axis('tight')
plt.legend(loc='best', scatterpoints=1, fontsize=5)
plt.savefig('ytsne', format='pdf', dpi=600)
plt.show()
fig2 = plt.figure()
ax2 = fig2.add_subplot(111)
colors = cm.Spectral(np.linspace(0, 1, num_classes))
xx = embeddings[:, 0]
yy = embeddings[:, 1]
for i in range(num_classes):
ax2.scatter(xx[y_sample_pred == i],
yy[y_sample_pred == i],
color=colors[i],
label=labels[i],
s=10)
ax2.xaxis.set_major_formatter(NullFormatter())
ax2.yaxis.set_major_formatter(NullFormatter())
plt.axis('tight')
plt.legend(loc='best', scatterpoints=1, fontsize=5)
plt.savefig('y_pred_tsne', format='pdf', dpi=600)
plt.show()
def main(config):
np.random.seed(config.random_seed)
# ensure directories are setup
prepare_dirs(config)
# load data
X_train, y_train, _, _ = load_data(config.data_dir)
# shuffle dataset
if config.shuffle:
p = np.random.permutation(len(X_train))
X_train = X_train[p]
y_train = y_train[p]
num_classes = len(np.unique(y_train))
labels = np.arange(num_classes)
# restrict to a sample because slow
mask = np.arange(config.num_samples)
X_sample = X_train[mask].squeeze()
y_sample = y_train[mask]
# grab file names for saving
file_name = name_file(config) + '.p'
v = '_v1'
if config.with_images == True:
v = '_v2'
img_name = name_file(config) + v + '.pdf'
if config.compute_embeddings:
print("X_sample: {}".format(X_sample.shape))
print("y_sample: {}".format(y_sample.shape))
# flatten images to (N, D) for feeding to t-SNE
X_sample_flat = np.reshape(X_sample, [X_sample.shape[0], -1])
# compute tsne embeddings
embeddings = TSNE(n_components=config.num_dimensions, init='pca', verbose=2).fit_transform(X_sample_flat)
# dump
pickle.dump(embeddings, open(config.data_dir + file_name, "wb"))
# else load
print("Loading embedding...")
embeddings = pickle.load(open(config.data_dir + file_name, "rb"))
print('Plotting...')
if config.num_dimensions == 3:
# safeguard
if config.with_images == True:
sys.exit("Cannot plot images with 3D plots.")
fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
colors = cm.Spectral(np.linspace(0, 1, num_classes))
xx = embeddings[:, 0]
yy = embeddings[:, 1]
zz = embeddings[:, 2]
# plot the 3D data points
for i in range(num_classes):
ax.scatter(xx[y_sample==i], yy[y_sample==i], zz[y_sample==i], color=colors[i], label=labels[i], s=10)
ax.xaxis.set_major_formatter(NullFormatter())
ax.yaxis.set_major_formatter(NullFormatter())
ax.zaxis.set_major_formatter(NullFormatter())
plt.axis('tight')
plt.legend(loc='best', scatterpoints=1, fontsize=5)
plt.savefig(config.plot_dir + img_name, format='pdf', dpi=600)
plt.show()
# 2D plot
else:
fig = plt.figure()
ax = fig.add_subplot(111)
colors = cm.Spectral(np.linspace(0, 1, num_classes))
xx = embeddings[:, 0]
yy = embeddings[:, 1]
# plot the images
if config.with_images == True:
for i, (x, y) in enumerate(zip(xx, yy)):
im = OffsetImage(X_sample[i], zoom=0.1, cmap='gray')
ab = AnnotationBbox(im, (x, y), xycoords='data', frameon=False)
ax.add_artist(ab)
ax.update_datalim(np.column_stack([xx, yy]))
ax.autoscale()
# plot the 2D data points
for i in range(num_classes):
ax.scatter(xx[y_sample==i], yy[y_sample==i], color=colors[i], label=labels[i], s=10)
ax.xaxis.set_major_formatter(NullFormatter())
ax.yaxis.set_major_formatter(NullFormatter())
plt.axis('tight')
plt.legend(loc='best', scatterpoints=1, fontsize=5)
plt.savefig(config.plot_dir + img_name, format='pdf', dpi=600)
plt.show()
plotDir2 = '../plots/gws/Ka2017x2_FixZPT0/u_g_r_i_z_y_J_H_K/0_14/ejecta/GW170817/1.00/' pcklFile = os.path.join(plotDir2, "data.pkl") f = open(pcklFile, 'r') (data_out, data2, tmag2, lbol2, mag2, t0_best2, zp_best2, n_params2, labels2, best2, truths2) = pickle.load(f) f.close() tmag1 = tmag1 + t0_best1 tmag2 = tmag2 + t0_best2 title_fontsize = 30 label_fontsize = 30 filts = ["u", "g", "r", "i", "z", "y", "J", "H", "K"] #colors=cm.jet(np.linspace(0,1,len(filts))) colors = cm.Spectral(np.linspace(0, 1, len(filts)))[::-1] magidxs = [0, 1, 2, 3, 4, 5, 6, 7, 8] tini, tmax, dt = 0.0, 21.0, 0.1 tt = np.arange(tini, tmax, dt) color2 = 'coral' color1 = 'cornflowerblue' plotName = "%s/data_panels.pdf" % (plotDir) #plt.figure(figsize=(20,18)) plt.figure(figsize=(20, 28)) tini, tmax, dt = 0.0, 21.0, 0.1 tt = np.arange(tini, tmax, dt) cnt = 0
def visualize(args):
num_layers, layer_id = None, 0
while True:
# extract X, labels <= create tsne input
X, y = [], []
id2lab, lab2id = {}, {}
with open(args.feat_file, 'r') as f:
for line in f:
info = json.loads(line.strip())
span_start = np.array(info['start_layer'][layer_id],
dtype=np.float32)
span_end = np.array(info['end_layer'][layer_id],
dtype=np.float32)
label = info['label']
if label not in lab2id:
lab2id[label] = len(lab2id)
id2lab[lab2id[label]] = label
y.append(lab2id[label])
X.append(
np.concatenate(
(span_start, span_end,
np.multiply(span_start,
span_end), span_start - span_end)))
if not num_layers:
num_layers = len(info['end_layer'])
X = np.array(X, dtype=np.float32)
y = np.array(y, dtype=np.int)
layer_id += 1
# perform t-SNE
embeddings = TSNE(n_components=2,
init='pca',
verbose=0,
perplexity=30,
n_iter=500).fit_transform(X)
xx = embeddings[:, 0]
yy = embeddings[:, 1]
# plot
fig = plt.figure()
ax = fig.add_subplot(111)
num_classes = len(lab2id)
colors = cm.Spectral(np.linspace(0, 1, num_classes))
labels = np.arange(num_classes)
for i in range(num_classes):
ax.scatter(xx[y == i],
yy[y == i],
color=colors[i],
label=id2lab[i],
s=12)
ax.xaxis.set_major_formatter(NullFormatter())
ax.yaxis.set_major_formatter(NullFormatter())
plt.axis('tight')
box = ax.get_position()
ax.set_position([box.x0, box.y0, box.width * 0.8, box.height])
ax.legend(loc='center left', bbox_to_anchor=(1, 0.5))
plt.savefig(args.output_file_prefix + str(layer_id) + ".pdf",
format='pdf',
dpi=600)
#plt.show()
print('layer %d plot => %s' %
(layer_id, args.output_file_prefix + str(layer_id) + ".pdf"))
if layer_id == num_layers:
break
print(X.shape)
print(Y.shape)
# perform tsne
embeddings = TSNE(n_components=2, init='pca', verbose=2).fit_transform(X)
xx = embeddings[:, 0]
yy = embeddings[:, 1]
import matplotlib.pyplot as plt
from matplotlib.pyplot import cm
from matplotlib.offsetbox import OffsetImage, AnnotationBbox
from matplotlib.ticker import NullFormatter
fig = plt.figure()
ax = fig.add_subplot(111)
num_classes = 5
colors = cm.Spectral(np.linspace(0, 1, num_classes))
labels = np.arange(num_classes)
# plot the 2D data points
for i in range(num_classes):
ax.scatter(xx[Y==i], yy[Y==i], color=colors[i], label=labels[i], s=10)
ax.xaxis.set_major_formatter(NullFormatter())
ax.yaxis.set_major_formatter(NullFormatter())
plt.axis('tight')
plt.legend(loc='best', scatterpoints=1, fontsize=5)
plt.savefig(args.out_dir + '/' + args.embed_dir.split("/")[-1] +".pdf", format='pdf', dpi=600)
plt.show()
def get_colors():
return [iter(cm.GnBu(np.linspace(0,1,len(ALPHAS[scale])+skip))), \
iter(cm.YlOrBr(np.linspace(0,1,len(ALPHAS[scale])+skip))),\
iter(cm.YlGn(np.linspace(0,1,len(ALPHAS[scale])+skip))),\
iter(cm.Spectral(np.linspace(0,1,len(ALPHAS[scale])+skip))), ] #http://matplotlib.org/users/colormaps.html
def do_knfit(df,
outputDir='./knfit',
errorbudget=1.0,
n_live_points=100,
evidence_tolerance=0.5):
if len(df.index) < 2:
print('Not enough data for candidate %s... continuing.',
df.candname.values[0])
return
if not os.path.isdir(outputDir):
os.makedirs(outputDir)
candname = df.name.values[0].decode()
filters = list(set(df.filtname))
mint = 0.0
maxt = 7.0
dt = 0.05
tt = np.arange(mint, maxt, dt)
doWaveformExtrapolate = True
ZPRange = 5.0
T0Range = 2.0
ModelPath = "../gwemlightcurves/output/svdmodels/"
T0 = np.inf
mag_min = np.inf
data_out = {}
for index, row in df.iterrows():
filt = row.filtname
mag = row.magpsf
dmag = row.sigmapsf
mjd = Time(row.jd, format='jd').mjd
magzp = row.magzpsci
if 99.0 == mag:
mag = magzp
dmag = np.inf
else:
T0 = np.min([T0, mjd])
mag_min = np.min([mag_min, mag])
if not filt in data_out:
data_out[filt] = np.empty((0, 3), float)
data_out[filt] = np.append(data_out[filt],
np.array([[mjd, mag, dmag]]),
axis=0)
distmod = mag_min - -16
distance = 10**((distmod / 5.0) + 1.0) / 1e6
for ii, key in enumerate(data_out.keys()):
if key == "t":
continue
else:
data_out[key][:, 0] = data_out[key][:, 0] - T0
data_out[key][:, 1] = data_out[key][:, 1] - 5 * (
np.log10(distance * 1e6) - 1)
for ii, key in enumerate(data_out.keys()):
if key == "t":
continue
else:
idxs = np.intersect1d(
np.where(data_out[key][:, 0] >= mint)[0],
np.where(data_out[key][:, 0] <= maxt)[0])
data_out[key] = data_out[key][idxs, :]
for ii, key in enumerate(data_out.keys()):
idxs = np.where(~np.isnan(data_out[key][:, 2]))[0]
if key == "t":
continue
else:
data_out[key] = data_out[key][idxs, :]
for ii, key in enumerate(data_out.keys()):
if not key in filters:
del data_out[key]
for ii, key in enumerate(data_out.keys()):
if ii == 0:
samples = data_out[key].copy()
else:
samples = np.vstack((samples, data_out[key].copy()))
idx = np.argmin(samples[:, 0])
samples = samples[idx, :]
Global.data_out = data_out
Global.errorbudget = errorbudget
Global.ZPRange = ZPRange
Global.T0Range = T0Range
Global.doLightcurves = 1
Global.filters = filters
Global.doWaveformExtrapolate = doWaveformExtrapolate
modelfile = os.path.join(ModelPath, 'Bu2019inc_mag.pkl')
with open(modelfile, 'rb') as handle:
svd_mag_model = pickle.load(handle)
Global.svd_mag_model = svd_mag_model
modelfile = os.path.join(ModelPath, 'Bu2019inc_lbol.pkl')
with open(modelfile, 'rb') as handle:
svd_lbol_model = pickle.load(handle)
Global.svd_lbol_model = svd_lbol_model
plotDir = os.path.join(outputDir, candname)
if not os.path.isdir(plotDir):
os.makedirs(plotDir)
max_iter = -1
best = []
parameters = ["t0", "mej", "phi", "theta", "zp"]
labels = [
r"$T_0$", r"${\rm log}_{10} (M_{\rm ej})$", r"$\Phi$", r"$\Theta$",
"ZP"
]
n_params = len(parameters)
pymultinest.run(myloglike_Bu2019inc_ejecta,
myprior_Bu2019inc_ejecta,
n_params,
importance_nested_sampling=False,
resume=True,
verbose=True,
sampling_efficiency='parameter',
n_live_points=n_live_points,
outputfiles_basename='%s/2-' % plotDir,
evidence_tolerance=evidence_tolerance,
multimodal=False,
max_iter=max_iter)
multifile = lightcurve_utils.get_post_file(plotDir)
data = np.loadtxt(multifile)
t0, mej, phi, theta, zp, loglikelihood = data[:,
0], 10**data[:,
1], data[:,
2], data[:,
3], data[:,
4], data[:,
5]
idx = np.argmax(loglikelihood)
t0_best, mej_best, phi_best, theta_best, zp_best = data[idx, 0], 10**data[
idx, 1], data[idx, 2], data[idx, 3], data[idx, 4]
zp_mu, zp_std = 0.0, Global.ZPRange
zp_best = scipy.stats.norm(zp_mu, zp_std).ppf(zp_best)
tmag, lbol, mag = Bu2019inc_model_ejecta(mej_best, phi_best, theta_best)
pcklFile = os.path.join(plotDir, "data.pkl")
f = open(pcklFile, 'wb')
pickle.dump((data_out, data, tmag, lbol, mag, t0_best, zp_best, n_params,
labels, best), f)
f.close()
title_fontsize = 30
label_fontsize = 30
plotName = "%s/corner.pdf" % (plotDir)
figure = corner.corner(data[:, :-1],
labels=labels,
quantiles=[0.16, 0.5, 0.84],
show_titles=True,
title_kwargs={"fontsize": title_fontsize},
label_kwargs={"fontsize": label_fontsize},
title_fmt=".2f",
smooth=3,
color="coral")
figure.set_size_inches(14.0, 14.0)
plt.savefig(plotName)
plt.close()
tmag = tmag + t0_best
#colors=cm.rainbow(np.linspace(0,1,len(filters)))
colors = cm.Spectral(np.linspace(0, 1, len(filters)))[::-1]
color2 = 'coral'
color1 = 'cornflowerblue'
plotName = "%s/models_panels.pdf" % (plotDir)
#plt.figure(figsize=(20,18))
plt.figure(figsize=(20, 28))
cnt = 0
for filt, color in zip(filters, colors):
cnt = cnt + 1
if cnt == 1:
ax1 = plt.subplot(len(filters), 1, cnt)
else:
ax2 = plt.subplot(len(filters), 1, cnt, sharex=ax1, sharey=ax1)
if not filt in data_out: continue
samples = data_out[filt]
t, y, sigma_y = samples[:, 0], samples[:, 1], samples[:, 2]
idx = np.where(~np.isnan(y))[0]
t, y, sigma_y = t[idx], y[idx], sigma_y[idx]
if len(t) == 0: continue
idx = np.where(np.isfinite(sigma_y))[0]
plt.errorbar(t[idx],
y[idx],
sigma_y[idx],
fmt='o',
c=color,
markersize=16,
label='%s-band' % filt)
idx = np.where(~np.isfinite(sigma_y))[0]
plt.errorbar(t[idx],
y[idx],
sigma_y[idx],
fmt='v',
c=color,
markersize=16)
magave = lightcurve_utils.get_mag(mag, filt)
ii = np.where(~np.isnan(magave))[0]
f = interp.interp1d(tmag[ii], magave[ii], fill_value='extrapolate')
maginterp = f(tt)
#plt.plot(tt,maginterp+zp_best,'--',c=color,linewidth=3)
#plt.fill_between(tt,maginterp+zp_best-errorbudget,maginterp+zp_best+errorbudget,facecolor=color,alpha=0.2)
plt.plot(tt, maginterp + zp_best, '--', c=color2, linewidth=3)
plt.fill_between(tt,
maginterp + zp_best - errorbudget,
maginterp + zp_best + errorbudget,
facecolor=color2,
alpha=0.2)
plt.ylabel('%s' % filt, fontsize=48, rotation=0, labelpad=40)
plt.xlim([0.0, 10.0])
plt.ylim([-22.0, -8.0])
plt.gca().invert_yaxis()
plt.grid()
if cnt == 1:
ax1.set_yticks([-22, -18, -14, -10])
plt.setp(ax1.get_xticklabels(), visible=False)
#l = plt.legend(loc="upper right",prop={'size':36},numpoints=1,shadow=True, fancybox=True)
elif not cnt == len(filters):
plt.setp(ax2.get_xticklabels(), visible=False)
plt.xticks(fontsize=36)
plt.yticks(fontsize=36)
ax1.set_zorder(1)
plt.xlabel('Time [days]', fontsize=48)
plt.tight_layout()
plt.savefig(plotName)
plt.close()
return
allPred_nr = np.zeros((len(allRes['images']),n_class))
for i,k in enumerate(allRes['images']):
alLabels_nr[i,allResNonRob['images'][k]['real_label']] = 1
allPred_nr[i,:] = allResNonRob['images'][k]['real_scores']
ap_r, mrec_r, mprec_r = computeAP(allPred,alLabels)
ap_nr, mrec_nr, mprec_nr = computeAP(allPred_nr,alLabels)
n_high= 15
print ap_r.mean(), ap_nr.mean()
mdiff = np.argsort(np.abs(ap_r - ap_nr))[::-1]
print('Score: || %s |'%(' | '.join(['%6s'%selected_attrs[mdiff[i]][:6] for i in xrange(n_high)])))
print('AP_R : || %s |'%(' | '.join([' %.2f' % ap_r[mdiff[i]] for i in xrange(n_high)])))
print('AP_NR: || %s |'%(' | '.join([' %.2f' % ap_nr[mdiff[i]] for i in xrange(n_high)])))
color=cm.Spectral(np.linspace(0,1,n_high))
for i in xrange(n_high):
#pr,rec,th = precision_recall_curve(alLabels[:,mdiff[i]],allPred[:,mdiff[i]]);
plt.plot(mrec_r[:,mdiff[i]],mprec_r[:,mdiff[i]],label=selected_attrs[mdiff[i]]+'_r',c=color[i])
for i in xrange(n_high):
#pr,rec,th = precision_recall_curve(alLabels_nr[:,mdiff[i]],allPred_nr[:,mdiff[i]]);
plt.plot(mrec_nr[:,mdiff[i]],mprec_nr[:,mdiff[i]],label=selected_attrs[mdiff[i]]+'_nr',linestyle=':',c=color[i])
plt.legend(ncol=2);
plt.xlabel('Recall'); plt.ylabel('Precision')
plt.title('Top %d classes with largest difference in ap'%(n_high))
plt.show()
isp = (alLabels[:,0] == 1.)
ap_r_wp, mrec_r_wp, mprec_r_wp = computeAP(allPred[isp,1:],alLabels[isp,1:])
ap_r_wop, mrec_r_wop, mprec_r_wop = computeAP(allPred[~isp,1:],alLabels[~isp,1:])
