def plot_and_test(args, xarr=np.linspace(-25,25,1000), moments=moments, nsigcut=None):
arrargs = np.array([0] + list(args))
gg = gaussian(xarr, *args)
noise = np.random.randn(1000)
moments_nonoise = np.array(moments(xarr,gg,veryverbose=True))
fmtstring = "h: %6.2f a: %6.2f x: %6.2f w: %6.2f "
print("No noise")
print("Moments: "+fmtstring % tuple(moments_nonoise) )
print("Difference: "+fmtstring % tuple(moments_nonoise-arrargs))
pylab.subplot(2,2,1)
pylab.plot(xarr,gg,'k')
pylab.plot(xarr,gaussian(xarr,*moments_nonoise[1:]),'r')
print("diff^2: %0.2f" % (((gg-gaussian(xarr,*moments_nonoise[1:]))**2).sum()))
def plottest_withnoise(noiseamp, spnum, xarr=xarr, gg=gg, noise=noise):
fitspec = gg+noiseamp*noise
moments_noise1 = np.array(moments(xarr,fitspec,veryverbose=True,nsigcut=nsigcut))
print("Noise = %i%% of peak" % (round(noiseamp/args[0]*100)))
print("Moments: "+fmtstring % tuple(moments_noise1) )
print("Difference: "+fmtstring % tuple(moments_noise1-arrargs))
pylab.subplot(2,2,spnum)
pylab.plot(xarr,fitspec,'k')
pylab.plot(xarr,gaussian(xarr,*moments_noise1[1:]),'r')
print("chi^2: %0.2f" % ( (((fitspec-gaussian(xarr,*moments_noise1[1:]))**2) / ((noise*noiseamp)**2)).sum() / len(noise) ),)
print("correct chi^2: %0.2f" % ( (((fitspec-gg)**2) / ((noise*noiseamp)**2)).sum() / len(noise) ))
for noiseamp,spnum in zip((0.5,2,5),(2,3,4)):
plottest_withnoise(noiseamp, spnum)
def hfonly_66_fixed(x, dv, amp1, amp2, width, **kwargs):
dx1 = 26.9
dx2 = 31.4
return (gaussian(x, amp1, dv+dx1, width)+
gaussian(x, amp1, dv-dx1, width)+
gaussian(x, amp2, dv+dx2, width)+
gaussian(x, amp2, dv-dx2, width))
def plottest_withnoise(noiseamp, spnum, xarr=xarr, gg=gg, noise=noise):
fitspec = gg+noiseamp*noise
moments_noise1 = np.array(moments(xarr,fitspec,veryverbose=True,nsigcut=nsigcut))
print("Noise = %i%% of peak" % (round(noiseamp/args[0]*100)))
print("Moments: "+fmtstring % tuple(moments_noise1) )
print("Difference: "+fmtstring % tuple(moments_noise1-arrargs))
pylab.subplot(2,2,spnum)
pylab.plot(xarr,fitspec,'k')
pylab.plot(xarr,gaussian(xarr,*moments_noise1[1:]),'r')
print("chi^2: %0.2f" % ( (((fitspec-gaussian(xarr,*moments_noise1[1:]))**2) / ((noise*noiseamp)**2)).sum() / len(noise) ),)
print("correct chi^2: %0.2f" % ( (((fitspec-gg)**2) / ((noise*noiseamp)**2)).sum() / len(noise) ))
def toy_observation(snr=2,
debug=False,
seed=0,
v1=10,
v2=70,
nchan=1000,
truth_narrow=[4, 40, 1.0],
truth_wide=[2, 41, 3.0]):
np.random.seed(seed)
if debug: # <CheatMode>
log.setLevel('DEBUG')
# initialize the spectral axis
xunit, bunit = 'km/s', 'K'
refX = 120 * u.GHz
log.debug("Genarating a spactral axis instance from {}"
" to {} {}".format(v1, v2, xunit))
xarr = SpectroscopicAxis(np.linspace(v1, v2, nchan) * u.Unit(xunit),
refX=refX,
velocity_convention='radio')
# generate a spectrum approximated by a gaussian
log.debug("Gaussian parameters for the"
" narrow component: {}".format(truth_narrow))
log.debug("Gaussian parameters for the"
" wide component: {}".format(truth_wide))
true_data_narrow = gaussian(xarr, *truth_narrow)
true_data_wide = gaussian(xarr, *truth_wide)
true_total = true_data_narrow + true_data_wide
signal_peak = true_total.max()
log.debug("For a signal-to-noise ratio of {} the square root of noise"
" variance is {:.2f} {}.".format(snr, signal_peak / snr, bunit))
noise = np.random.normal(loc=0, scale=signal_peak / snr, size=xarr.size)
observed = true_total + noise
log.setLevel('INFO') # <\CheatMode>
# make a spectrum class instance in Tmb units
xarr._make_header()
sp = Spectrum(xarr=xarr, data=observed, unit=u.Unit(bunit), header={})
sp.header['NPEAKS'] = 2
sp.header['NOISERMS'] = round(signal_peak / snr, 4)
for comp, name in zip([truth_narrow, truth_wide], ['1', '2']):
sp.header['AMP_' + name] = comp[0]
sp.header['XOFF_' + name] = comp[1]
sp.header['SIG_' + name] = comp[2]
return sp
def plottest_withnoise(noiseamp, spnum, xarr=xarr, gg=gg, noise=noise):
fitspec = gg + noiseamp * noise
moments_noise1 = np.array(
moments(xarr, fitspec, veryverbose=True, nsigcut=nsigcut))
print("Noise = %i%% of peak" % (round(noiseamp / args[0] * 100)))
print("Moments: " + fmtstring % tuple(moments_noise1))
print("Difference: " + fmtstring % tuple(moments_noise1 - arrargs))
pylab.subplot(2, 2, spnum)
pylab.plot(xarr, fitspec, 'k')
pylab.plot(xarr, gaussian(xarr, *moments_noise1[1:]), 'r')
print(
"chi^2: %0.2f" %
((((fitspec - gaussian(xarr, *moments_noise1[1:]))**2) /
((noise * noiseamp)**2)).sum() / len(noise)), )
print("correct chi^2: %0.2f" %
((((fitspec - gg)**2) /
((noise * noiseamp)**2)).sum() / len(noise)))
def hfonly(x, dv, dx1, amp1, width1, dx2, amp2, width2, return_hyperfine_components=False, **kwargs):
if return_hyperfine_components:
return (gaussian(x, amp1, dv+dx1, width1),
gaussian(x, amp1, dv-dx1, width1),
gaussian(x, amp2, dv+dx2, width2),
gaussian(x, amp2, dv-dx2, width2))
else:
return (gaussian(x, amp1, dv+dx1, width1)+
gaussian(x, amp1, dv-dx1, width1)+
gaussian(x, amp2, dv+dx2, width2)+
gaussian(x, amp2, dv-dx2, width2))
def plot_and_test(args,
xarr=np.linspace(-25, 25, 1000),
moments=moments,
nsigcut=None):
arrargs = np.array([0] + list(args))
gg = gaussian(xarr, *args)
noise = np.random.randn(1000)
moments_nonoise = np.array(moments(xarr, gg, veryverbose=True))
fmtstring = "h: %6.2f a: %6.2f x: %6.2f w: %6.2f "
print("No noise")
print("Moments: " + fmtstring % tuple(moments_nonoise))
print("Difference: " + fmtstring % tuple(moments_nonoise - arrargs))
pylab.subplot(2, 2, 1)
pylab.plot(xarr, gg, 'k')
pylab.plot(xarr, gaussian(xarr, *moments_nonoise[1:]), 'r')
print("diff^2: %0.2f" %
(((gg - gaussian(xarr, *moments_nonoise[1:]))**2).sum()))
def plottest_withnoise(noiseamp, spnum, xarr=xarr, gg=gg, noise=noise):
fitspec = gg + noiseamp * noise
moments_noise1 = np.array(
moments(xarr, fitspec, veryverbose=True, nsigcut=nsigcut))
print("Noise = %i%% of peak" % (round(noiseamp / args[0] * 100)))
print("Moments: " + fmtstring % tuple(moments_noise1))
print("Difference: " + fmtstring % tuple(moments_noise1 - arrargs))
pylab.subplot(2, 2, spnum)
pylab.plot(xarr, fitspec, 'k')
pylab.plot(xarr, gaussian(xarr, *moments_noise1[1:]), 'r')
print(
"chi^2: %0.2f" %
((((fitspec - gaussian(xarr, *moments_noise1[1:]))**2) /
((noise * noiseamp)**2)).sum() / len(noise)), )
print("correct chi^2: %0.2f" %
((((fitspec - gg)**2) /
((noise * noiseamp)**2)).sum() / len(noise)))
for noiseamp, spnum in zip((0.5, 2, 5), (2, 3, 4)):
plottest_withnoise(noiseamp, spnum)
def generate_cubes(nCubes=90, nBorder=1, noise_rms=0.1, fix_vlsr=False,
random_seed=None, remove_low_sep=False, remove_high_sep=False, remove_low_snr=False, set_name='GAS_reg', regression=False, plot=False):
"""
This places nCubes random cubes into the specified output directory
"""
#if not os.path.isdir(output_dir):
# os.mkdir(output_dir)
cube_km = SpectralCube.read('random_cube_NH3_11_0.fits')
xarr11 = cube_km.with_spectral_unit(u.km / u.s, velocity_convention='radio').spectral_axis.value[250:-250]
xarr11 = numpy.flip(xarr11, axis=0) #flip so negative VLSR on left
#xarr11 = spaxis((np.linspace(-500, 499, 500) * 5.72e-6
# + nh3con.freq_dict['oneone'] / 1e9),
# unit='GHz',
# refX=nh3con.freq_dict['oneone'] / 1e9,
# velocity_convention='radio', refX_unit='GHz')
#xarr22 = spaxis((np.linspace(-500, 499, 1000) * 5.72e-6
# + nh3con.freq_dict['twotwo'] / 1e9), unit='GHz',
# refX=nh3con.freq_dict['twotwo'] / 1e9,
# velocity_convention='radio', refX_unit='GHz')
out_arr = []
out_vels = []
nDigits = int(np.ceil(np.log10(nCubes)))
if random_seed:
np.random.seed(random_seed)
# Check that ncubes divisible by 3
if (nCubes % 3 != 0):
print 'Warning: Number of cubes not divisible by 3'
print 'Setting ncubes to 90'
nCubes = 90
# Ensure output is balanced between each class
nComps = np.concatenate((np.ones(nCubes/3).astype(int), np.zeros(nCubes/3).astype(int), np.zeros(nCubes/3).astype(int)+2))
if regression:
nComps = np.ones(nCubes)+1
out_y1 = np.where(np.array(nComps)==1, 1, 0)
out_y2 = np.where(np.array(nComps)==0, 1, 0)
out_y3 = np.where(np.array(nComps)==2, 1, 0)
mu_range_spectral = [-10, 10]
sigma_range_spectral = [0.15, 0.8]
#Width1NT = 0.1 * np.exp(1.5 * np.random.randn(nCubes))
#Width2NT = 0.1 * np.exp(1.5 * np.random.randn(nCubes))
#sigma1 = np.sqrt(Width1NT + 0.08**2)
#sigma2 = np.sqrt(Width2NT + 0.08**2)
#peak_range_spectral = [0.4, 6.0]
mn = numpy.random.uniform(0.05, 0.25, size=nCubes) #rms noise added .01 - 0.1 # 0.05 - 0.2
Voff1 = numpy.random.uniform(mu_range_spectral[0], mu_range_spectral[1], size=(nCubes))
sigma1 = numpy.random.uniform(sigma_range_spectral[0], sigma_range_spectral[1], size=(nCubes))
#peak1 = numpy.random.uniform(peak_range_spectral[0], peak_range_spectral[1], size=(nCubes))
#peak1 = numpy.random.uniform(0.25*3, 0.05*40, size=(nCubes))
#peak1 = mn*3 + np.random.rand(nCubes) * mn*20
peak1 = numpy.ones(nCubes)
xoff = np.random.randint(2, size=nCubes)
xoff[xoff==0]=-1
sigma2 = numpy.random.uniform(sigma_range_spectral[0], sigma_range_spectral[1], size=(nCubes))
Voff2 = Voff1 + numpy.max(numpy.column_stack((sigma1, sigma2)),axis=1)*numpy.random.uniform(1.5, 5, size=(nCubes))*xoff
#Voff2 = numpy.random.uniform(mu_range_spectral[0], mu_range_spectral[1], size=(nCubes))
#peak2 = numpy.random.uniform(peak_range_spectral[0], peak_range_spectral[1], size=(nCubes))
#peak2 = numpy.random.uniform(0.25*3, 0.05*40, size=(nCubes))
#peak2 = mn*3 + np.random.rand(nCubes) * mn*20
peak2 = numpy.random.uniform(2*mn, 1.0, size=(nCubes))
#print numpy.where(numpy.absolute(peak1-peak2)<mn)
#print np.where(np.abs(Voff1-Voff2)<np.max(np.column_stack((Width1, Width2)), axis=1))[0]
if remove_low_sep:
# Find where centroids are too close
too_close = np.where(np.abs(Voff1-Voff2)<np.max(np.column_stack((sigma1, sigma2)), axis=1))
# Move the centroids farther apart by the length of largest line width
min_Voff = np.min(np.column_stack((Voff2[too_close],Voff1[too_close])), axis=1)
max_Voff = np.max(np.column_stack((Voff2[too_close],Voff1[too_close])), axis=1)
Voff1[too_close]=min_Voff-np.max(np.column_stack((sigma1[too_close], sigma2[too_close])), axis=1)/2.
Voff2[too_close]=max_Voff+np.max(np.column_stack((sigma1[too_close], sigma2[too_close])), axis=1)/2.
if remove_high_sep:
# Find where centroids are too close
too_far = np.where(np.abs(Voff1-Voff2)>5.0)
# Move the centroids farther apart by the length of largest line width
Voff2[too_far]=Voff1[too_far]-sigma2[too_far]*numpy.random.uniform(2, 3, size=(len(too_far)))
if remove_low_snr:
low_snr = np.where(np.min(np.column_stack((peak1,peak2)), axis=1)/mn<4.0)
mn[low_snr] = np.min(np.column_stack((peak1,peak2)), axis=1)[low_snr]/4.0
#print Voff1[too_close]
#print Voff2[too_close]
#print len(np.where(np.abs(Voff1-Voff2)<np.max(np.column_stack((Width1, Width2)), axis=1))[0])
# Normalize centroids between -1 and 1
Voff1_norm = 2*((Voff1-min(xarr11))/(max(xarr11)-min(xarr11)))-1
Voff2_norm = 2*((Voff2-min(xarr11))/(max(xarr11)-min(xarr11)))-1
# Normalize dispersion to be number of channels
step = abs(xarr11[23]-xarr11[24])
sigma1_norm = sigma1/step
sigma2_norm = sigma2/step
scale = np.array([[0.05, 0.05, 0.05]]) # Tpeak, Sig, Voff
gradX1 = np.random.randn(nCubes, 3) * scale
gradY1 = np.random.randn(nCubes, 3) * scale
gradX2 = np.random.randn(nCubes, 3) * scale
gradY2 = np.random.randn(nCubes, 3) * scale
for i in ProgressBar(range(nCubes)):
xmat, ymat = np.indices((2 * nBorder + 1, 2 * nBorder + 1))
cube11 = np.zeros((xarr11.shape[0], 2 * nBorder + 1, 2 * nBorder + 1))
for xx, yy in zip(xmat.flatten(), ymat.flatten()):
T1 = peak1[i] * (1 + gradX1[i, 0] * (xx - 1)
+ gradY1[i, 0] * (yy - 1))
T2 = peak2[i] * (1 + gradX2[i, 0] * (xx - 1)
+ gradY2[i, 0] * (yy - 1))
W1 = np.abs(sigma1[i] * (1 + gradX1[i, 1] * (xx - 1)
+ gradY1[i, 1] * (yy - 1)))
W2 = np.abs(sigma2[i] * (1 + gradX2[i, 1] * (xx - 1)
+ gradY2[i, 1] * (yy - 1)))
V1 = Voff1[i] + (gradX2[i, 2] * (xx - 1) + gradY2[i, 2] * (yy - 1))
V2 = Voff2[i] + (gradX2[i, 2] * (xx - 1) + gradY2[i, 2] * (yy - 1))
if nComps[i] == 1:
spec11 = ig.gaussian(xarr11, T1,V1,W1)
#spec22 = ammonia.cold_ammonia(xarr22, T1,
# ntot=N1,
# width=W1,
# xoff_v=V1)
if (xx == nBorder) and (yy == nBorder):
Tmax11a = np.max(spec11)
Tmax11b = 0
Tmax11 = np.max(spec11)
if nComps[i] == 2:
spec11a = ig.gaussian(xarr11, T1,V1,W1)
spec11b = ig.gaussian(xarr11, T2,V2,W2)
spec11 = spec11a + spec11b
if (xx == nBorder) and (yy == nBorder):
Tmax11a = peak1[i]
Tmax11b = peak2[i]
Tmax11 = np.max(spec11)
#plt.plot(xarr11, spec11)
#plt.scatter([Voff1[i], Voff2[i]], [T1,T2], color='red', alpha=0.5)
#plt.show()
if nComps[i]==0:
cube11[:, yy, xx] = np.zeros(500)
else:
cube11[:, yy, xx] = spec11
#cube22[:, yy, xx] = spec22
if nComps[i]!=0:
Tmax11a /= np.max(cube11)
Tmax11b /= np.max(cube11)
cube11 /= np.max(cube11)
cube11 += np.random.randn(*cube11.shape) * mn[i]
#cube22 += np.random.randn(*cube22.shape) * noise_rms
#loc11 = cube11[:,1,1].reshape(500,1)
#loc11 = loc11/np.max(loc11)
#glob11 = np.mean(cube11.reshape(500,9),axis=1)
#glob11 = glob11/np.max(glob11)
loc11 = cube11[:,1,1].reshape(500,1)
T1 = Tmax11a/np.max(loc11)
T2 = Tmax11b/np.max(loc11)
loc11 = loc11/np.max(loc11)
glob11 = np.mean(cube11.reshape(500,9),axis=1)
glob11 = glob11/np.max(glob11)
z = np.column_stack((loc11,glob11))
out_arr.append(z)
V1 = Voff1_norm[i]
V2 = Voff2_norm[i]
W1 = sigma1_norm[i]
W2 = sigma2_norm[i]
out_vel = [min([V1,V2]), max([V1,V2]), [W1,W2][np.argmin([V1,V2])], [W1,W2][np.argmax([V1,V2])], [T1,T2][np.argmin([V1,V2])], [T1,T2][np.argmax([V1,V2])]]
out_vels.append(out_vel)
if plot and nComps[i]==2:
if nComps[i]==1:
tt = 'One-Component'
elif nComps[i]==2:
tt = 'Two-Component'
else:
tt = 'Noise-Only'
plt.plot(xarr11, loc11) #, label='central pixel (local)'
plt.plot(xarr11, glob11+1) #, label='3x3 average (global)'
plt.errorbar([Voff1[i], Voff2[i]], [0,0.1], xerr=[sigma1[i],sigma2[i]], color='orange', fmt='none')
plt.scatter([Voff1[i], Voff2[i]], [T1,T2], color='red', alpha=0.5)
plt.scatter([Voff1[i], Voff2[i]], [T1+1,T2+1], color='red', alpha=0.5)
plt.xlabel('Synthetic $V_{LSR}$', size=14)
plt.ylabel('Normalized Intensity', size=14)
#plt.legend(fontsize=14)
plt.title(tt, size=14)
plt.show()
#xxx = numpy.linspace(-1,1,500)
#step = abs(xxx[1]-xxx[2])
#plt.plot(xxx, loc11) #, label='central pixel (local)'
#plt.plot(xxx, glob11+1) #, label='3x3 average (global)'
#plt.errorbar([Voff1_norm[i], Voff2_norm[i]], [0,0.1], xerr=[sigma1_norm[i]*step,sigma2_norm[i]*step], color='orange', fmt='none')
#plt.scatter([Voff1_norm[i], Voff2_norm[i]], [T1,T2], color='red', alpha=0.5)
#plt.show()
with h5py.File('three_class_' + set_name + '.h5', 'w') as hf:
hf.create_dataset('data', data=np.array(out_arr))
hf.close()
with h5py.File('labels_three_class_' + set_name + '.h5', 'w') as hf:
hf.create_dataset('data', data=np.column_stack((out_y1, out_y2, out_y3)))
hf.close()
with h5py.File('params_three_class_' + set_name + '.h5', 'w') as hf:
hf.create_dataset('data', data=np.array(out_vels))
hf.close()