def priorSample(ngauss=128, quantile=0.5, iter='8th', survey='2MASS', dataFilename='All.npz', Nsamples=1.2e6, xdgmmFilename='xdgmm.fit', xlabel='X', ylabel='Y', contourColor='k'):
setup_text_plots(fontsize=16, usetex=True)
xdgmm = XDGMM(filename=xdgmmFilename)
figPrior = plt.figure(figsize=(12, 5.5))
figPrior.subplots_adjust(left=0.1, right=0.95,
bottom=0.15, top=0.95,
wspace=0.1, hspace=0.1)
sample = xdgmm.sample(Nsamples)
negParallax = sample[:,1] < 0
nNegP = np.sum(negParallax)
while nNegP > 0:
sampleNew = xdgmm.sample(nNegP)
sample[negParallax] = sampleNew
negParallax = sample[:,1] < 0
nNegP = np.sum(negParallax)
samplex = sample[:,0]
sampley = testXD.absMagKinda2absMag(sample[:,1])
ax3 = figPrior.add_subplot(121)
alpha = 0.1
xlim = [-0.25, 1.25]
ylim = [6, -6]
levels = 1.0 - np.exp(-0.5 * np.arange(1.0, 2.1, 1.0) ** 2)
corner.hist2d(samplex, sampley, ax=ax3, levels=levels, bins=200, plot_datapoints=False, no_fill_contours=True, plot_density=False, color=contourColor)
ax3.scatter(samplex, sampley, s=1, lw=0, c='k', alpha=alpha)
ax4 = figPrior.add_subplot(122)
for i in range(xdgmm.n_components):
points = drawEllipse.plotvector(xdgmm.mu[i], xdgmm.V[i])
ax4.plot(points[0, :], testXD.absMagKinda2absMag(points[1,:]), 'k-', alpha=xdgmm.weights[i]/np.max(xdgmm.weights))
titles = ["Extreme Deconvolution\n resampling",
"Extreme Deconvolution\n cluster locations"]
ax = [ax3, ax4]
for i in range(2):
ax[i].set_xlim(xlim)
ax[i].set_ylim(ylim[0], ylim[1]*1.1)
ax[i].text(0.05, 0.95, titles[i],
ha='left', va='top', transform=ax[i].transAxes, fontsize=18)
ax[i].set_xlabel(xlabel, fontsize = 18)
if i in (1, 3):
ax[i].yaxis.set_major_formatter(plt.NullFormatter())
else:
ax[i].set_ylabel(ylabel, fontsize = 18)
figPrior.savefig('prior_ngauss' + str(ngauss) +'.png')
def dataViz(survey='2MASS',
ngauss=128,
quantile=0.05,
dataFilename='All.npz',
iter='10th',
Nsamples=3e5,
contourColor='k',
dustFile='dust.npz',
sdss5=False):
if survey == 'APASS':
mag1 = 'B'
mag2 = 'V'
absmag = 'G'
xlabel = 'B-V'
ylabel = r'M$_\mathrm{G}$'
xlim = [-0.2, 2]
ylim = [9, -2]
if survey == '2MASS':
mag1 = 'J'
mag2 = 'K'
absmag = 'J'
xlabel = r'$(J-K)^C$'
ylabel = r'$M_J^C$'
xlim = [-0.25, 1.25]
ylim = [6, -6]
xdgmmFilename = 'xdgmm.' + str(ngauss) + 'gauss.dQ' + str(
quantile) + '.' + iter + '.' + survey + '.' + dataFilename + '.fit'
tgas, twoMass, Apass, bandDictionary, indices = testXD.dataArrays()
dustEBV = 0.0
color = testXD.colorArray(mag1, mag2, dustEBV, bandDictionary)
absMagKinda, apparentMagnitude = testXD.absMagKindaArray(
absmag, dustEBV, bandDictionary, tgas['parallax'])
color_err = np.sqrt(
bandDictionary[mag1]['array'][bandDictionary[mag1]['err_key']]**2. +
bandDictionary[mag2]['array'][bandDictionary[mag2]['err_key']]**2.)
absMagKinda_err = tgas['parallax_error'] * 10.**(
0.2 * bandDictionary[absmag]['array'][bandDictionary[absmag]['key']])
xdgmm = XDGMM(filename=xdgmmFilename)
sample = xdgmm.sample(Nsamples)
negParallax = sample[:, 1] < 0
nNegP = np.sum(negParallax)
while nNegP > 0:
sampleNew = xdgmm.sample(nNegP)
sample[negParallax] = sampleNew
negParallax = sample[:, 1] < 0
nNegP = np.sum(negParallax)
positive = absMagKinda > 0
y = absMagKinda[positive]
yplus = y + absMagKinda_err[positive]
yminus = y - absMagKinda_err[positive]
parallaxErrGoesNegative = yminus < 0
absMagYMinus = testXD.absMagKinda2absMag(yminus)
absMagYMinus[parallaxErrGoesNegative] = -50.
yerr_minus = testXD.absMagKinda2absMag(y) - absMagYMinus
yerr_plus = testXD.absMagKinda2absMag(yplus) - testXD.absMagKinda2absMag(y)
#yerr_minus = testXD.absMagKinda2absMag(yplus) - testXD.absMagKinda2absMag(y)
#yerr_plus = testXD.absMagKinda2absMag(y) - absMagYMinus
"""
testfig, testax = plt.subplots(3)
testax[0].scatter(testXD.absMagKinda2absMag(y), y, s=1)
testax[0].set_xlabel('absMag')
testax[0].set_ylabel('absMagKinda')
testax[1].scatter(testXD.absMagKinda2absMag(y), absMagYMinus, s=1)
testax[1].set_xlabel('absMag')
testax[1].set_ylabel('absMag Minus')
testax[2].scatter(testXD.absMagKinda2absMag(y), testXD.absMagKinda2absMag(yplus), s=1)
testax[2].set_xlabel('absMag')
testax[2].set_ylabel('absMag Plus')
plt.show()
"""
dp.plot_sample(color[positive],
testXD.absMagKinda2absMag(y),
sample[:, 0],
testXD.absMagKinda2absMag(sample[:, 1]),
xdgmm,
xerr=color_err[positive],
yerr=[yerr_minus, yerr_plus],
xlabel=xlabel,
ylabel=ylabel,
xlim=xlim,
ylim=ylim,
errSubsample=2.4e3,
thresholdScatter=2.,
binsScatter=200,
contourColor=contourColor)
dataFile = 'data_noDust.pdf'
priorFile = 'prior_' + str(ngauss) + 'gauss.pdf'
os.rename('plot_sample.data.pdf', dataFile)
os.rename('plot_sample.prior.pdf', priorFile)
#import pdb; pdb.set_trace()
data = np.load(dustFile)
dustEBV = data['ebv']
color = testXD.colorArray(mag1, mag2, dustEBV, bandDictionary)
absMagKinda, apparentMagnitude = testXD.absMagKindaArray(
absmag, dustEBV, bandDictionary, tgas['parallax'])
cNorm = plt.matplotlib.colors.Normalize(vmin=-6, vmax=2)
posteriorFile = 'posteriorParallax.' + str(ngauss) + 'gauss.dQ' + str(
quantile) + '.' + iter + '.' + survey + '.' + dataFilename
for file in [posteriorFile, 'posteriorSimple.npz']:
data = np.load(file)
parallax = data['mean']
parallax_err = np.sqrt(data['var'])
notnans = ~np.isnan(parallax) & ~np.isnan(parallax_err)
parallax = parallax[notnans]
parallax_err = parallax_err[notnans]
apparentMagnitudeGood = apparentMagnitude[notnans]
c = np.log(data['var']) - np.log(tgas['parallax_error']**2.)
absMagKinda = parallax * 10.**(0.2 * apparentMagnitudeGood)
absMagKinda_err = parallax_err * 10.**(0.2 * apparentMagnitudeGood)
y = absMagKinda
yplus = y + absMagKinda_err
yminus = y - absMagKinda_err
parallaxErrGoesNegative = yminus < 0
absMagYMinus = testXD.absMagKinda2absMag(yminus)
absMagYMinus[parallaxErrGoesNegative] = -50.
absMag = testXD.absMagKinda2absMag(y)
yerr_minus = absMag - absMagYMinus
yerr_plus = testXD.absMagKinda2absMag(yplus) - absMag
#notnan = ~np.isnan(color[notnans]) & ~np.isnan(absMag)
contourColor = 'k'
ascii.write([
color[notnans], absMag, color_err[notnans], yerr_minus, yerr_plus,
c[notnans]
],
'cmdExpectation.txt',
names=[
'color', 'absMag', 'color_err', 'absMag_errMinus',
'absMag_errPlus', 'logDeltaVar'
])
dp.plot_sample(color[notnans],
absMag,
sample[:, 0],
testXD.absMagKinda2absMag(sample[:, 1]),
xdgmm,
xerr=color_err[notnans],
yerr=[yerr_minus, yerr_plus],
xlabel=xlabel,
ylabel=ylabel,
xlim=xlim,
ylim=ylim,
errSubsample=1.2e3,
thresholdScatter=2.,
binsScatter=200,
c=c,
norm=cNorm,
cmap='Blues',
contourColor=contourColor,
posterior=True,
sdss5=sdss5,
rasterized=False)
dataFile = 'inferredDistances_data_' + file.split('.')[0] + '.pdf'
priorFile = 'prior_' + str(ngauss) + 'gauss.pdf'
os.rename('plot_sample.data.pdf', dataFile)
os.rename('plot_sample.prior.pdf', priorFile)
def dataViz(survey='2MASS', ngauss=128, quantile=0.05, dataFilename='All.npz', iter='10th', Nsamples=3e5, contourColor='k', dustFile='dust.npz', sdss5=False, whatsThatFeature=False):
if survey == 'APASS':
mag1 = 'B'
mag2 = 'V'
absmag = 'G'
xlabel='B-V'
ylabel = r'M$_\mathrm{G}$'
xlim = [-0.2, 2]
ylim = [9, -2]
if survey == '2MASS':
mag1 = 'J'
mag2 = 'K'
absmag = 'J'
xlabel = r'$(J-K)^C$'
ylabel = r'$M_J^C$'
xlim = [-0.25, 1.25]
ylim = [6, -6]
xdgmmFilename = 'xdgmm.' + str(ngauss) + 'gauss.dQ' + str(quantile) + '.' + iter + '.' + survey + '.' + dataFilename + '.fit'
tgas, twoMass, Apass, bandDictionary, indices = testXD.dataArrays()
dustEBV = 0.0
color = testXD.colorArray(mag1, mag2, dustEBV, bandDictionary)
absMagKinda, apparentMagnitude = testXD.absMagKindaArray(absmag, dustEBV, bandDictionary, tgas['parallax'])
color_err = np.sqrt(bandDictionary[mag1]['array'][bandDictionary[mag1]['err_key']]**2. + bandDictionary[mag2]['array'][bandDictionary[mag2]['err_key']]**2.)
absMagKinda_err = tgas['parallax_error']*10.**(0.2*bandDictionary[absmag]['array'][bandDictionary[absmag]['key']])
xdgmm = XDGMM(filename=xdgmmFilename)
sample = xdgmm.sample(Nsamples)
negParallax = sample[:,1] < 0
nNegP = np.sum(negParallax)
while nNegP > 0:
sampleNew = xdgmm.sample(nNegP)
sample[negParallax] = sampleNew
negParallax = sample[:,1] < 0
nNegP = np.sum(negParallax)
positive = absMagKinda > 0
y = absMagKinda[positive]
yplus = y + absMagKinda_err[positive]
yminus = y - absMagKinda_err[positive]
parallaxErrGoesNegative = yminus < 0
absMagYMinus = testXD.absMagKinda2absMag(yminus)
absMagYMinus[parallaxErrGoesNegative] = -50.
yerr_minus = testXD.absMagKinda2absMag(y) - absMagYMinus
yerr_plus = testXD.absMagKinda2absMag(yplus) - testXD.absMagKinda2absMag(y)
#yerr_minus = testXD.absMagKinda2absMag(yplus) - testXD.absMagKinda2absMag(y)
#yerr_plus = testXD.absMagKinda2absMag(y) - absMagYMinus
"""
testfig, testax = plt.subplots(3)
testax[0].scatter(testXD.absMagKinda2absMag(y), y, s=1)
testax[0].set_xlabel('absMag')
testax[0].set_ylabel('absMagKinda')
testax[1].scatter(testXD.absMagKinda2absMag(y), absMagYMinus, s=1)
testax[1].set_xlabel('absMag')
testax[1].set_ylabel('absMag Minus')
testax[2].scatter(testXD.absMagKinda2absMag(y), testXD.absMagKinda2absMag(yplus), s=1)
testax[2].set_xlabel('absMag')
testax[2].set_ylabel('absMag Plus')
plt.show()
"""
dp.plot_sample(color[positive], testXD.absMagKinda2absMag(y), sample[:,0], testXD.absMagKinda2absMag(sample[:,1]),
xdgmm, xerr=color_err[positive], yerr=[yerr_minus, yerr_plus], xlabel=xlabel, ylabel=ylabel, xlim=xlim, ylim=ylim, errSubsample=2.4e3, thresholdScatter=2., binsScatter=200, contourColor=contourColor)
dataFile = 'data_noDust.pdf'
priorFile = 'prior_' + str(ngauss) +'gauss.pdf'
os.rename('plot_sample.data.pdf', dataFile)
os.rename('plot_sample.prior.pdf', priorFile)
#import pdb; pdb.set_trace()
data = np.load(dustFile)
dustEBV = data['ebv']
color = testXD.colorArray(mag1, mag2, dustEBV, bandDictionary)
absMagKinda, apparentMagnitude = testXD.absMagKindaArray(absmag, dustEBV, bandDictionary, tgas['parallax'])
cNorm = plt.matplotlib.colors.Normalize(vmin=-6, vmax=2)
posteriorFile = 'posteriorParallax.' + str(ngauss) + 'gauss.dQ' + str(quantile) + '.' + iter + '.' + survey + '.' + dataFilename
for file in [posteriorFile]:#, 'posteriorSimple.npz']:
data = np.load(file)
parallax = data['mean']
parallax_err = np.sqrt(data['var'])
notnans = ~np.isnan(parallax) & ~np.isnan(parallax_err)
parallax = parallax[notnans]
parallax_err = parallax_err[notnans]
apparentMagnitudeGood = apparentMagnitude[notnans]
c = np.log(data['var']) - np.log(tgas['parallax_error']**2.)
absMagKinda = parallax*10.**(0.2*apparentMagnitudeGood)
absMagKinda_err = parallax_err*10.**(0.2*apparentMagnitudeGood)
y = absMagKinda
yplus = y + absMagKinda_err
yminus = y - absMagKinda_err
parallaxErrGoesNegative = yminus < 0
absMagYMinus = testXD.absMagKinda2absMag(yminus)
absMagYMinus[parallaxErrGoesNegative] = -50.
absMag = testXD.absMagKinda2absMag(y)
yerr_minus = absMag - absMagYMinus
yerr_plus = testXD.absMagKinda2absMag(yplus) - absMag
#notnan = ~np.isnan(color[notnans]) & ~np.isnan(absMag)
contourColor = 'k'
ascii.write([color[notnans], absMag, color_err[notnans], yerr_minus, yerr_plus, c[notnans]], 'cmdExpectation.txt', names=['color', 'absMag', 'color_err', 'absMag_errMinus', 'absMag_errPlus', 'logDeltaVar'])
if whatsThatFeature & (file == posteriorFile):
figFeature, axFeature = plt.subplots()
x = color[notnans]
y = absMag
#levels = 1.0 - np.exp(-0.5 * np.arange(1.0, 2.1, 1.0) ** 2)
im = corner.hist2d(x, y, ax=axFeature, levels=None, bins=200, no_fill_contours=True, plot_density=False, color=contourColor, rasterized=True, plot_contours=False)
axFeature.set_xlim(xlim)
axFeature.set_ylim(ylim)
axFeature.set_xlabel(xlabel)
axFeature.set_ylabel(ylabel)
lowerMainSequence = (0.45, 5.5)
upperMainSequence = (-0.225, 2)
binarySequence = (0.75, 4)
redClump = (0.35, -2)
redGiantBranch = (1.0, -2)
turnOff = (0.0, 3.5)
features = [lowerMainSequence, upperMainSequence, binarySequence, redClump, redGiantBranch, turnOff]
labels = ['lower MS', 'upper MS', 'binary sequence', 'red clump', 'RGB', 'MS turn off']
for l, f in zip(labels, features): axFeature.text(f[0], f[1], l, fontsize=15)
figFeature.savefig('whatsThatFeature.pdf', format='pdf')
class XDGMMTestCase(unittest.TestCase):
"TestCase class for XDGMM class."
def setUp(self):
"""
Set up each test with a new XDGMM object and some data.
"""
self.xdgmm = XDGMM(n_components=3)
self.files = []
"""
Use scikit-learn GaussianMixture for sampling some data points
"""
self.gmm = skl_GMM(n_components=3,
max_iter=10,
covariance_type='full',
random_state=None)
self.gmm.weights_ = np.array([0.3, 0.5, 0.2])
self.gmm.means_ = np.array(
[np.array([0, 1]),
np.array([5, 4]),
np.array([2, 4])])
self.gmm.covariances_ = np.array([
np.diag((2, 1)),
np.array([[1, 0.2], [0.2, 1]]),
np.diag((0.3, 0.5))
])
self.gmm.precisions_ = np.linalg.inv(self.gmm.covariances_)
self.gmm.precisions_cholesky_ = np.linalg.cholesky(
self.gmm.precisions_)
self.X = self.gmm.sample(1000)[0]
errs = 0.2 * np.random.random_sample((1000, 2))
self.Xerr = np.zeros(self.X.shape + self.X.shape[-1:])
diag = np.arange(self.X.shape[-1])
self.Xerr[:, diag, diag] = np.vstack([errs[:, 0]**2, errs[:, 1]**2]).T
def tearDown(self):
"""
Clean up files saved by tests
"""
for fname in self.files:
os.remove('test.fit')
def test_Fit(self):
this_mu = self.xdgmm.mu
this_V = self.xdgmm.V
this_weights = self.xdgmm.weights
self.xdgmm.fit(self.X, self.Xerr)
self.assertIsNotNone(self.xdgmm.mu)
self.assertIsNotNone(self.xdgmm.V)
self.assertIsNotNone(self.xdgmm.weights)
def test_Sample(self):
self.xdgmm.fit(self.X, self.Xerr)
sam = self.xdgmm.sample(1000)
self.assertEqual(sam.shape, (1000, 2))
def test_Score(self):
self.xdgmm.fit(self.X, self.Xerr)
data = np.array([np.array([0, 2]), np.array([4, 4])])
err = np.array([np.diag((0.2, 0.1)), np.diag((0.15, 0.15))])
self.assertNotEqual(self.xdgmm.score(data, err), 0)
def test_ReadWrite(self):
self.xdgmm.fit(self.X, self.Xerr)
self.xdgmm.save_model('test.fit')
xd2 = XDGMM(filename='test.fit')
self.assertLess(self.xdgmm.mu[0, 0] - xd2.mu[0, 0], 1e-5)
self.assertLess(self.xdgmm.V[0, 0, 0] - xd2.V[0, 0, 0], 1e-5)
self.assertLess(self.xdgmm.weights[0] - xd2.weights[0], 1e-5)
self.files.append('test.fit')
def test_Condition(self):
self.xdgmm.fit(self.X, self.Xerr)
cond_xd = self.xdgmm.condition(X_input=np.array([np.nan, 3.5]))
self.assertEqual(cond_xd.mu.shape, (3, 1))
self.assertEqual(cond_xd.V.shape, (3, 1, 1))