def ensemble_predictions(members, testX, nproc):
if nproc>6:
nproc=6
def para_predict(modname):
from keras.models import load_model
model = load_model(modname)
prediction = model.predict(testX)
return prediction
queue = pprocess.Queue(limit=nproc)
calc = queue.manage(pprocess.MakeParallel(para_predict))
yhats = []
for i in members:
calc(i)
#yhats.append(para_predict(i,testX))
for preds in queue:
yhats.append(preds)
# make predictions
#yhats = [model.predict(testX) for model in members]
yhats = numpy.array(yhats)
# weighted sum across ensemble members
#summed = numpy.tensordot(yhats, weights, axes=((0),(0)))
summed = numpy.sum(yhats, axis=0)
# argmax across classes
result = numpy.argmax(summed, axis=1)
return result
def rebase_multi(filename, nproc=8, mask_percent=0.4, blorder_max=3, window_size=31): """ Returns a baseline-subtracted cube. Can be run with parallel processes. filename = name of datacube to process (including its path) nproc = number of parallel processes desired mask_percent = percentage of pixels to select for baseline fitting blorder_max = largest order polynomial to fit (fit from blorder_max down to order of 1) """ cube = SpectralCube.read(filename) queue = pprocess.Queue(limit=nproc, continuous=1) calc = queue.manage(rebase) tic = time.time() # create cube to store rebaselined data cube_out = np.zeros(cube.shape) * np.nan pixels = cube.shape[1] * cube.shape[2] counter = 0 for i in np.array_split(range(cube.shape[1]), nproc): calc(i, data=cube, mask_percent=mask_percent, blorder_max=blorder_max, window_size=window_size) for i, j, ss in queue: cube_out[:,i,j]=ss counter+=1 print str(counter) + ' of ' + str(pixels) + ' pixels completed \r', sys.stdout.flush() print "\n %f s for parallel computation." % (time.time() - tic) cube_final = SpectralCube(data=cube_out, wcs=cube.wcs, header=cube.header) cube_final.write(filename[0:-5] + '_rebase_multi.fits', format='fits', overwrite=True)
def pp_wavelength_loop(cube,
head,
wavels,
out_cube,
AO,
psfvars,
adrvals,
newsize,
outspax,
adr_switch='ON',
usecpus=mp.cpu_count() - 1):
'''Function to take input datacube and process it iteratively through
each wavelength channel as follows:
- Generate PSF array for given channel
- Add effect of ADR to channel
- Convolve cube channel with PSF
- Frebin up to chosen output spaxel scale
Inputs:
cube: Datacube
head: Datacube header
wavels: Wavelength array
out_cube: Empty output cube
AO: AO mode [LTAO, SCAO, Gaussian]
psdvars: list containing [psfparams, psfspax, psfsize,
[D,eps], res_jitter, seeing]
adrvals: array of ADR values
newsize: tuple containing (x_newsize, y_newsize) array sizes
outspax: tuple containing (x, y) output spaxel scales
adr_switch: On or OFF. Turns ADR effect on or off
usecpus: no. of CPUs to use
Output:
cube: Processed cube
head: Updated header
inspax: Input spaxel scale (mas, mas)
outspax: Output spaxel scale (mas, mas)
'''
import pprocess as pp
print 'Using ', usecpus, ' CPUs'
queue = pp.Queue(limit=usecpus)
waveloop = queue.manage(pp.MakeParallel(pp_wavelength_channel))
print "Initialising..."
for lam in xrange(len(wavels)):
waveloop(cube[lam, :, :], head, wavels[lam], lam, AO, psfvars,
adrvals[lam], newsize, outspax, adr_switch)
print "Processing..."
for chan, it in queue:
update_progress(n.round(it / float(len(wavels)), 2))
out_cube[it, :, :] = chan
return out_cube, head
def pp_wavelength_loop(cube,
head,
wavels,
out_cube,
newsize,
outspax,
usecpus=mp.cpu_count() - 1):
'''Function to take input datacube and process it iteratively through
each wavelength channel as follows:
- Generate PSF array for given channel
- Add effect of ADR to channel
- Convolve cube channel with PSF
- Frebin up to chosen output spaxel scale
Inputs:
cube: Datacube
head: Datacube header
wavels: Wavelength array
out_cube: Empty output cube
newsize: tuple containing (x_newsize, y_newsize) array sizes
outspax: tuple containing (x, y) output spaxel scales
usecpus: no. of CPUs to use
Output:
cube: Processed cube
head: Updated header
inspax: Input spaxel scale (mas, mas)
outspax: Output spaxel scale (mas, mas)
'''
import pprocess as pp
print 'Using ', usecpus, ' CPUs'
queue = pp.Queue(limit=usecpus)
waveloop = queue.manage(pp.MakeParallel(pp_wavelength_channel))
print "Initialising..."
for lam in xrange(len(wavels)):
waveloop(cube[lam, :, :], head, wavels[lam], lam, newsize, outspax)
print "Processing..."
for chan, it in queue:
update_progress(n.round(it / float(len(wavels)), 2))
out_cube[it, :, :] = chan
return out_cube, head
def gauss_fitter(region='Cepheus_L1251',
snr_min=3.0,
mol='C2S',
vmin=5.0,
vmax=10.0,
convolve=False,
use_old_conv=False,
multicore=1,
file_extension=None):
"""
Fit a Gaussian to non-NH3 emission lines from GAS.
It creates a cube for the best-fit Gaussian, a cube
for the best-fit Gaussian with noise added back into
the spectrum, and a parameter map of Tpeak, Vlsr, and FWHM
Parameters
----------
region : str
Name of region to reduce
snr_min : float
Lowest signal-to-noise pixels to include in the line-fitting
mol : str
name of molecule to fit
vmin : numpy.float
Minimum centroid velocity, in km/s.
vmax : numpy.float
Maximum centroid velocity, in km/s.
convolve : bool or float
If not False, specifies the beam-size to convolve the original map with
Beam-size must be given in arcseconds
use_old_conv : bool
If True, use an already convolved map with name:
region + '_' + mol + file_extension + '_conv.fits'
This convolved map must be in units of km/s
multicore : int
Maximum number of simultaneous processes desired
file_extension: str
filename extension
"""
if file_extension:
root = file_extension
else:
# root = 'base{0}'.format(blorder)
root = 'all'
molecules = ['C2S', 'HC7N_22_21', 'HC7N_21_20', 'HC5N']
MolFile = '{0}/{0}_{2}_{1}.fits'.format(region, root, mol)
ConvFile = '{0}/{0}_{2}_{1}_conv.fits'.format(region, root, mol)
GaussOut = '{0}/{0}_{2}_{1}_gauss_cube.fits'.format(region, root, mol)
GaussNoiseOut = '{0}/{0}_{2}_{1}_gauss_cube_noise.fits'.format(
region, root, mol)
ParamOut = '{0}/{0}_{2}_{1}_param_cube.fits'.format(region, root, mol)
# Load the spectral cube and convert to velocity units
cube = SpectralCube.read(MolFile)
cube_km = cube.with_spectral_unit(u.km / u.s, velocity_convention='radio')
# If desired, convolve map with larger beam
# or load previously created convolved cube
if convolve:
cube = SpectralCube.read(MolFile)
cube_km_1 = cube.with_spectral_unit(u.km / u.s,
velocity_convention='radio')
beam = radio_beam.Beam(major=convolve * u.arcsec,
minor=convolve * u.arcsec,
pa=0 * u.deg)
cube_km = cube_km_1.convolve_to(beam)
cube_km.write(ConvFile, format='fits', overwrite=True)
if use_old_conv:
cube_km = SpectralCube.read(ConvFile)
# Define the spectral axis in km/s
spectra_x_axis_kms = np.array(cube_km.spectral_axis)
# Find the channel range corresponding to vmin and vmax
# -- This is a hold-over from when I originally set up the code to
# use a channel range rather than velocity range.
# Can change later, but this should work for now.
low_channel = np.where(spectra_x_axis_kms <= vmax
)[0][0] + 1 # Add ones to change index to channel
high_channel = np.where(spectra_x_axis_kms >= vmin
)[0][-1] + 1 # Again, hold-over from older setup
peak_channels = [low_channel, high_channel]
# Create cubes for storing the fitted Gaussian profiles
# and the Gaussians with noise added back into the spectrum
header = cube_km.header
cube_gauss = np.array(cube_km.unmasked_data[:, :, :])
cube_gauss_noise = np.array(cube_km.unmasked_data[:, :, :])
shape = np.shape(cube_gauss)
# Set up a cube for storing fitted parameters
param_cube = np.zeros((6, shape[1], shape[2]))
param_header = cube_km.header
# Define the Gaussian profile
def p_eval(x, a, x0, sigma):
return a * np.exp(-(x - x0)**2 / (2 * sigma**2))
# Create some arrays full of NANs
# To be used in output cubes if fits fail
nan_array = np.empty(shape[0]) # For gauss cubes
nan_array[:] = np.NAN
nan_array2 = np.empty(param_cube.shape[0]) # For param cubes
nan_array2[:] = np.NAN
# Loop through each pixel and find those
# with SNR above snr_min
x = []
y = []
pixels = 0
for (i, j), value in np.ndenumerate(cube_gauss[0]):
spectra = np.array(cube_km.unmasked_data[:, i, j])
if (False in np.isnan(spectra)):
rms = np.nanstd(
np.append(spectra[0:(peak_channels[0] - 1)],
spectra[(peak_channels[1] + 1):len(spectra)]))
if (max(spectra[peak_channels[0]:peak_channels[1]]) /
rms) > snr_min:
pixels += 1
x.append(i)
y.append(j)
else:
cube_gauss[:, i, j] = nan_array
param_cube[:, i, j] = nan_array2
cube_gauss_noise[:, i, j] = nan_array
print str(pixels) + ' Pixels above SNR=' + str(snr_min)
# Define a Gaussian fitting function for each pixel
# i, j are the x,y coordinates of the pixel being fit
def pix_fit(i, j):
spectra = np.array(cube_km.unmasked_data[:, i, j])
# Use the peak brightness Temp within specified channel
# range as the initial guess for Gaussian height
max_ch = np.argmax(spectra[peak_channels[0]:peak_channels[1]])
Tpeak = spectra[peak_channels[0]:peak_channels[1]][max_ch]
# Use the velocity of the brightness Temp peak as
# initial guess for Gaussian mean
vpeak = spectra_x_axis_kms[peak_channels[0]:peak_channels[1]][max_ch]
rms = np.std(
np.append(spectra[0:(peak_channels[0] - 1)],
spectra[(peak_channels[1] + 1):len(spectra)]))
err1 = np.zeros(shape[0]) + rms
# Create a noise spectrum based on rms of off-line channels
# This will be added to best-fit Gaussian to obtain a noisy Gaussian
noise = np.random.normal(0., rms, len(spectra_x_axis_kms))
# Define initial guesses for Gaussian fit
guess = [Tpeak, vpeak, 0.3] # [height, mean, sigma]
try:
coeffs, covar_mat = curve_fit(p_eval,
xdata=spectra_x_axis_kms,
ydata=spectra,
p0=guess,
sigma=err1,
maxfev=500)
gauss = np.array(
p_eval(spectra_x_axis_kms, coeffs[0], coeffs[1], coeffs[2]))
noisy_gauss = np.array(
p_eval(spectra_x_axis_kms, coeffs[0], coeffs[1],
coeffs[2])) + noise
params = np.append(coeffs, (covar_mat[0][0]**0.5, covar_mat[1][1]**
0.5, covar_mat[2][2]**0.5))
# params = ['Tpeak', 'VLSR','sigma','Tpeak_err','VLSR_err','sigma_err']
# Don't accept fit if fitted parameters are non-physical or too uncertain
if (params[0] < 0.01) or (params[3] > 1.0) or (
params[2] < 0.05) or (params[5] > 0.5) or (params[4] >
0.75):
noisy_gauss = nan_array
gauss = nan_array
params = nan_array2
# Don't accept fit if the SNR for fitted spectrum is less than SNR threshold
#if max(gauss)/rms < snr_min:
# noisy_gauss = nan_array
# gauss = nan_array
# params = nan_array2
except RuntimeError:
noisy_gauss = nan_array
gauss = nan_array
params = nan_array2
return i, j, gauss, params, noisy_gauss
# Parallel computation:
nproc = multicore # maximum number of simultaneous processes desired
queue = pprocess.Queue(limit=nproc)
calc = queue.manage(pprocess.MakeParallel(pix_fit))
tic = time.time()
counter = 0
# Uncomment to see some plots of the fitted spectra
#for i,j in zip(x,y):
#pix_fit(i,j)
#plt.plot(spectra_x_axis_kms, spectra, color='blue', drawstyle='steps')
#plt.plot(spectra_x_axis_kms, gauss, color='red')
#plt.show()
#plt.close()
# Begin parallel computations
# Store the best-fit Gaussians and parameters
# in their correct positions in the previously created cubes
for i, j in zip(x, y):
calc(i, j)
for i, j, gauss_spec, parameters, noisy_gauss_spec in queue:
cube_gauss[:, i, j] = gauss_spec
param_cube[:, i, j] = parameters
cube_gauss_noise[:, i, j] = noisy_gauss_spec
counter += 1
print str(counter) + ' of ' + str(pixels) + ' pixels completed \r',
sys.stdout.flush()
print "\n %f s for parallel computation." % (time.time() - tic)
# Save final cubes
# These will be in km/s units.
# Spectra will have larger values to the left, lower values to right
cube_final_gauss = SpectralCube(data=cube_gauss,
wcs=cube_km.wcs,
header=cube_km.header)
cube_final_gauss.write(GaussOut, format='fits', overwrite=True)
cube_final_gauss_noise = SpectralCube(data=cube_gauss_noise,
wcs=cube_km.wcs,
header=cube_km.header)
cube_final_gauss_noise.write(GaussNoiseOut, format='fits', overwrite=True)
# Construct appropriate header for param_cube
param_header['NAXIS3'] = len(nan_array2)
param_header['WCSAXES'] = 3
param_header['CRPIX3'] = 1
param_header['CDELT3'] = 1
param_header['CRVAL3'] = 0
param_header['PLANE1'] = 'Tpeak'
param_header['PLANE2'] = 'VLSR'
param_header['PLANE3'] = 'sigma'
param_header['PLANE5'] = 'Tpeak_err'
param_header['PLANE6'] = 'VLSR_err'
param_header['PLANE7'] = 'sigma_err'
fits.writeto(ParamOut, param_cube, header=param_header, clobber=True)