def test_progress_bar_as_generator():
sum = 0
for x in console.ProgressBar(range(50)):
sum += x
assert sum == 1225
sum = 0
for x in console.ProgressBar(50):
sum += x
assert sum == 1225
def main(date, auto_confirm=False):
db = PanMongo()
seq_ids = db.observations.distinct(
'sequence_id', {'date': {'$gte': Time(date).datetime}})
imgs = [record['data']['file_path'] for record in db.observations.find(
{'sequence_id': {'$in': seq_ids}}, {'data.file_path': 1})]
dirs = set([img[0:img.rindex('/') - 1].replace('/var/panoptes/images/fields/', '')
for img in imgs])
if auto_confirm is False:
print("Found the following dirs for {}:".format(date))
pprint(dirs)
if input("Proceed (Y/n): ") == 'n':
return
for d in console.ProgressBar(dirs):
run_cmd = ['gsutil', '-mq', 'cp', '-r', '/var/panoptes/images/fields/{}/*.fz'.format(d),
'gs://panoptes-survey/PAN001/{}/'.format(d)]
try:
completed_process = subprocess.run(run_cmd, stdout=subprocess.PIPE)
if completed_process.returncode != 0:
print("Problem uploading")
print(completed_process.stdout)
except Exception as e:
print("Problem uploading: {}".format(e))
def MakeRoundBeam(incube, outfile=None, overwrite=True):
'''
This takes a FITS file or a SpectralCube and outputs
Parameters
----------
filename : `string` or `SpectralCube`
Input spectral cube
Returns
-------
cube : `SpectralCube`
'''
if isinstance(incube, str):
cube = SpectralCube.read(incube)
if isinstance(incube, VaryingResolutionSpectralCube):
cube = incube
if not isinstance(cube, VaryingResolutionSpectralCube):
warnings.warn("No information about multiple beams")
return (None)
beams = cube.beams
major_axes = np.array([bm.major.to(u.deg).value for bm in beams])
target_beamsize = np.array(major_axes.max())
target_beam = Beam(major=target_beamsize * u.deg,
minor=target_beamsize * u.deg,
pa=0.0 * u.deg)
print("Target beam is : {}".format(target_beam))
# Let's assume square pixels
pixsize = cube.wcs.pixel_scale_matrix[1, 1]
fwhm2sigma = np.sqrt(8 * np.log(2))
output = np.zeros(cube.shape)
with console.ProgressBar(cube.shape[0]) as bar:
for ii, plane in enumerate(cube.filled_data[:]):
this_beam = beams[ii]
conv_beam = target_beam - this_beam
majpix = conv_beam.major.value / pixsize / fwhm2sigma
minpix = conv_beam.minor.value / pixsize / fwhm2sigma
output[ii, :, :] = ftconvolve(plane,
major=majpix,
minor=minpix,
angle=conv_beam.pa.value)
bar.update()
hdr = copy.copy(cube.header)
hdr['CASAMBM'] = False
hdr['BMAJ'] = float(target_beam.major.value)
hdr['BMIN'] = float(target_beam.major.value)
hdr['BPA'] = 0.0
outcube = SpectralCube(output, cube.wcs, header=hdr)
if outfile:
outcube.write(outfile, overwrite=overwrite)
return None
return (outcube)
def griddata(pixPerBeam=3.0,
templateHeader=None,
gridFunction=jincGrid,
rootdir='/lustre/pipeline/scratch/GAS/',
region='NGC1333',
dirname='NGC1333_NH3_11',
startChannel=762, endChannel=3584,
doBaseline=True,
baselineRegion=[slice(762, 1024, 1), slice(3072, 3584, 1)],
blorder=1,
Sessions=None,
file_extension=None,
rebase=False, **kwargs):
if not Sessions:
filelist = glob.glob(rootdir + '/' + region + '/' + dirname + '/*fits')
if not file_extension:
file_extension = '_all'
history_message = 'Gridding of data using all sessions'
else:
filelist = []
for scan_i in Sessions:
filelist.extend(glob.glob(rootdir + '/' + region +
'/' + dirname + '/*_sess' +
str(scan_i) + '.fits'))
if isinstance(Sessions, list):
if not file_extension:
file_extension = '_sess{0}-sess{1}'.format(Sessions[0],
Sessions[-1])
if (Sessions[-1] + 1. - Sessions[0]) / len(Sessions) == 1.0:
history_message = 'Gridding of data using sessions' \
'between {0} and {1}'.format(Sessions[0], Sessions[-1])
else:
history_message = 'Gridding of data using sessions: '
for scan_i in Sessions:
history_message += '{0}, '.format(scan_i)
else:
if not file_extension:
file_extension = '_sess{0}'.format(Sessions)
history_message = 'Gridding of data using session'\
'{0}'.format(Sessions)
if len(filelist) == 0:
warnings.warn('There are no FITS files to process '
'in ' + rootdir + '/' + region + '/' + dirname)
return
# check that every file in the filelist is valid
# If not then remove it and send warning message
for file_i in filelist:
try:
fits.open(file_i)
except:
warnings.warn('file {0} is corrupted'.format(file_i))
filelist.remove(file_i)
# pull a test structure
s = fits.getdata(filelist[0])
c = 299792458.
nu0 = s[0]['RESTFREQ']
Data_Unit = s[0]['TUNIT7']
beamSize = 1.22 * (c / nu0 / 100.0) * 180 / np.pi # in degrees
naxis3 = len(s[0]['DATA'][startChannel:endChannel])
# Default behavior is to park the object velocity at
# the center channel in the VRAD-LSR frame
crval3 = s[0]['RESTFREQ'] * (1 - s[0]['VELOCITY'] / c)
crpix3 = s[0]['CRPIX1'] - startChannel
ctype3 = s[0]['CTYPE1']
cdelt3 = s[0]['CDELT1']
w = wcs.WCS(naxis=3)
w.wcs.restfrq = nu0
w.wcs.radesys = s[0]['RADESYS']
w.wcs.equinox = s[0]['EQUINOX']
# We are forcing this conversion to make nice cubes.
w.wcs.specsys = 'LSRK'
w.wcs.ssysobs = 'TOPOCENT'
if templateHeader is None:
wcsdict = autoHeader(filelist, beamSize=beamSize,
pixPerBeam=pixPerBeam)
w.wcs.crpix = [wcsdict['CRPIX1'], wcsdict['CRPIX2'], crpix3]
w.wcs.cdelt = np.array([wcsdict['CDELT1'], wcsdict['CDELT2'], cdelt3])
w.wcs.crval = [wcsdict['CRVAL1'], wcsdict['CRVAL2'], crval3]
w.wcs.ctype = [wcsdict['CTYPE1'], wcsdict['CTYPE2'], ctype3]
naxis2 = wcsdict['NAXIS2']
naxis1 = wcsdict['NAXIS1']
else:
w.wcs.crpix = [templateHeader['CRPIX1'],
templateHeader['CRPIX2'], crpix3]
w.wcs.cdelt = np.array([templateHeader['CDELT1'],
templateHeader['CDELT2'], cdelt3])
w.wcs.crval = [templateHeader['CRVAL1'],
templateHeader['CRVAL2'], crval3]
w.wcs.ctype = [templateHeader['CTYPE1'],
templateHeader['CTYPE2'], ctype3]
naxis2 = templateHeader['NAXIS2']
naxis1 = templateHeader['NAXIS1']
outCube = np.zeros((int(naxis3), int(naxis2), int(naxis1)))
outWts = np.zeros((int(naxis2), int(naxis1)))
xmat, ymat = np.meshgrid(np.arange(naxis1), np.arange(naxis2),
indexing='ij')
xmat = xmat.reshape(xmat.size)
ymat = ymat.reshape(ymat.size)
xmat = xmat.astype(np.int)
ymat = ymat.astype(np.int)
ctr = 0
for thisfile in filelist:
ctr += 1
s = fits.open(thisfile)
print("Now processing {0}".format(thisfile))
print("This is file {0} of {1}".format(ctr, len(filelist)))
nuindex = np.arange(len(s[1].data['DATA'][0]))
for spectrum in console.ProgressBar((s[1].data)):
# Generate Baseline regions
baselineIndex = np.concatenate([nuindex[ss]
for ss in baselineRegion])
specData = spectrum['DATA']
# baseline fit
if doBaseline & np.all(np.isfinite(specData)):
specData = baselineSpectrum(specData, order=blorder,
baselineIndex=baselineIndex)
# This part takes the TOPOCENTRIC frequency that is at
# CRPIX1 (i.e., CRVAL1) and calculates the what frequency
# that would have in the LSRK frame with freqShiftValue.
# This then compares to the desired frequency CRVAL3.
DeltaNu = freqShiftValue(spectrum['CRVAL1'],
-spectrum['VFRAME']) - crval3
DeltaChan = DeltaNu / cdelt3
specData = channelShift(specData, -DeltaChan)
outslice = (specData)[startChannel:endChannel]
spectrum_wt = np.isfinite(outslice).astype(np.float)
outslice = np.nan_to_num(outslice)
xpoints, ypoints, zpoints = w.wcs_world2pix(spectrum['CRVAL2'],
spectrum['CRVAL3'],
spectrum['CRVAL1'], 0)
tsys = spectrum['TSYS']
if (tsys > 10) and (xpoints > 0) and (xpoints < naxis1) \
and (ypoints > 0) and (ypoints < naxis2):
pixelWeight, Index = gridFunction(xmat, ymat,
xpoints, ypoints,
pixPerBeam=pixPerBeam)
vector = np.outer(outslice * spectrum_wt,
pixelWeight / tsys**2)
wts = pixelWeight / tsys**2
outCube[:, ymat[Index], xmat[Index]] += vector
outWts[ymat[Index], xmat[Index]] += wts
# Temporarily do a file write for every batch of scans.
outWtsTemp = np.copy(outWts)
outWtsTemp.shape = (1,) + outWtsTemp.shape
outCubeTemp = np.copy(outCube)
outCubeTemp /= outWtsTemp
hdr = fits.Header(w.to_header())
hdr = addHeader_nonStd(hdr, beamSize, Data_Unit)
#
hdu = fits.PrimaryHDU(outCubeTemp, header=hdr)
hdu.writeto(dirname + file_extension + '.fits', clobber=True)
outWts.shape = (1,) + outWts.shape
outCube /= outWts
# Create basic fits header from WCS structure
hdr = fits.Header(w.to_header())
# Add non standard fits keyword
hdr = addHeader_nonStd(hdr, beamSize, Data_Unit)
# Adds history message
hdr.add_history(history_message)
hdr.add_history('Using GAS pipeline version {0}'.format(__version__))
hdu = fits.PrimaryHDU(outCube, header=hdr)
hdu.writeto(dirname + file_extension + '.fits', clobber=True)
w2 = w.dropaxis(2)
hdr2 = fits.Header(w2.to_header())
hdu2 = fits.PrimaryHDU(outWts, header=hdr2)
hdu2.writeto(dirname + file_extension + '_wts.fits', clobber=True)
if rebase:
if 'NH3_11' in dirname:
baseline.rebaseline(dirname + file_extension + '.fits',
windowFunction=baseline.ammoniaWindow,
line='oneone', **kwargs)
if 'NH3_22' in dirname:
winfunc = baseline.ammoniaWindow
baseline.rebaseline(dirname + file_extension + '.fits',
windowFunction=baseline.ammoniaWindow,
line='twotwo', **kwargs)
if 'NH3_33' in dirname:
baseline.rebaseline(dirname + file_extension + '.fits',
winfunc = baseline.ammoniaWindow,
line='threethree', **kwargs)
else:
baseline.rebaseline(dirname + file_extension + '.fits',
windowFunction=baseline.tightWindow,
**kwargs)
def test_zero_progress_bar():
with console.ProgressBar(0) as bar:
pass
def test_progress_bar2():
for x in console.ProgressBar(range(50)):
pass
def test_progress_bar():
# This stuff is hard to test, at least smoke test it
with console.ProgressBar(50) as bar:
for i in range(50):
bar.update()
def rebaseline(filename,
blorder=3,
baselineRegion=[slice(0, 800, 1),
slice(-800, 0, 1)],
windowFunction=None,
blankBaseline=False,
flagSpike=True,
v0=None,
VlsrByCoord=None,
verbose=False,
**kwargs):
"""
Rebaseline a data cube using robust regression of Legendre polynomials.
Parameters
----------
filename : string
FITS filename of the data cube
blorder : int
Order of the polynomial to fit to the data
baselineRegion : list
List of slices defining the default region of the spectrum, in
channels, to be used for the baseline fitting.
windowFunction : function
Name of function to be used that will accept spectrum data, and
velocity axis and will return a binary mask of the channels to be used
in the baseline fitting. Extra **kwargs are passed to windowFunction
to do with as it must.
blankBaseline : boolean
Blank the baseline region on a per-spectrum basis
Returns
-------
Nothing. A new FITS file is written out with the suffix 'rebaseN' where N
is the baseline order
"""
cube = SpectralCube.read(filename)
originalUnit = cube.spectral_axis.unit
cube = cube.with_spectral_unit(u.km / u.s, velocity_convention='radio')
spaxis = cube.spectral_axis.to(u.km / u.s).value
goodposition = np.isfinite(cube.apply_numpy_function(np.max, axis=0))
y, x = np.where(goodposition)
outcube = np.zeros(cube.shape) * np.nan
RegionName = (filename.split('/'))[-1]
RegionName = (RegionName.split('_'))[0]
nuindex = np.arange(cube.shape[0])
runmin = nuindex[-1]
runmax = nuindex[0]
if verbose:
pb = console.ProgressBar(len(y))
for thisy, thisx in zip(y, x):
spectrum = cube[:, thisy, thisx].value
if v0 is not None:
baselineIndex = windowFunction(spectrum, spaxis, v0=v0, **kwargs)
elif hasattr(windowFunction, '__call__') and \
hasattr(VlsrByCoord, '__call__'):
_, Dec, RA = cube.world[0, thisy, thisx]
# This determines a v0 appropriate for the region
v0 = VlsrByCoord(RA.value, Dec.value, RegionName, **kwargs)
baselineIndex = windowFunction(spectrum, spaxis, v0=v0, **kwargs)
else:
baselineIndex = np.zeros_like(spectrum, dtype=np.bool)
for ss in baselineRegion:
baselineIndex[ss] = True
runmin = np.min([nuindex[baselineIndex].min(), runmin])
runmax = np.max([nuindex[baselineIndex].max(), runmax])
# Use channel-to-channel difference as the noise value.
if flagSpike:
jumps = (spectrum - np.roll(spectrum, -1))
noise = mad1d(jumps) * 2**(-0.5)
baselineIndex *= (np.abs(jumps) < 5 * noise)
noise = mad1d(
(spectrum - np.roll(spectrum, -2))[baselineIndex]) * 2**(-0.5)
else:
noise = mad1d(
(spectrum - np.roll(spectrum, -2))[baselineIndex]) * 2**(-0.5)
if blankBaseline:
spectrum = robustBaseline(spectrum,
baselineIndex,
blorder=blorder,
noiserms=noise)
spectrum[baselineIndex] = np.nan
outcube[:, thisy, thisx] = spectrum
else:
outcube[:, thisy, thisx] = robustBaseline(spectrum,
baselineIndex,
blorder=blorder,
noiserms=noise)
if verbose:
pb.update()
outsc = SpectralCube(outcube,
cube.wcs,
header=cube.header,
meta={'BUNIT': cube.header['BUNIT']})
outsc = outsc[runmin:runmax, :, :] # cut beyond baseline edges
# Return to original spectral unit
outsc = outsc.with_spectral_unit(originalUnit)
outsc.write(filename.replace('.fits', '_rebase{0}.fits'.format(blorder)),
overwrite=True)
def griddata(pixPerBeam=3.0,
templateHeader=None,
gridFunction=jincGrid,
rootdir='/lustre/pipeline/scratch/GAS/',
region='NGC1333',
dirname='NGC1333_NH3_11',
startChannel=1024,
endChannel=3072,
doBaseline=True,
baselineRegion=[slice(512, 1024, 1),
slice(3072, 3584, 1)]):
filelist = glob.glob(rootdir + '/' + region + '/' + dirname + '/*fits')
#pull a test structure
s = fits.getdata(filelist[0])
c = 299792458.
nu0 = s[0]['RESTFREQ']
Data_Unit = s[0]['TUNIT7']
beamSize = 1.22 * (c / nu0 / 100.0) * 180 / np.pi # in degrees
naxis3 = len(s[0]['DATA'][startChannel:endChannel])
nuindex = np.arange(len(s[0]['DATA']))
baselineIndex = np.concatenate([nuindex[ss] for ss in baselineRegion])
nuslice = (range(naxis3))[startChannel:endChannel]
# Default behavior is to park the object velocity at the center channel in the VRAD-LSR frame
crval3 = s[0]['RESTFREQ'] * (1 - s[0]['VELOCITY'] / c)
crpix3 = s[0]['CRPIX1'] - startChannel
ctype3 = s[0]['CTYPE1']
cdelt3 = s[0]['CDELT1']
w = wcs.WCS(naxis=3)
w.wcs.restfrq = nu0
w.wcs.radesys = s[0]['RADESYS']
w.wcs.equinox = s[0]['EQUINOX']
w.wcs.specsys = 'LSRK' # We are forcing this conversion to make nice cubes.
w.wcs.ssysobs = 'TOPOCENT'
if templateHeader is None:
wcsdict = autoHeader(filelist,
beamSize=beamSize,
pixPerBeam=pixPerBeam)
w.wcs.crpix = [wcsdict['CRPIX1'], wcsdict['CRPIX2'], crpix3]
w.wcs.cdelt = np.array([wcsdict['CDELT1'], wcsdict['CDELT2'], cdelt3])
w.wcs.crval = [wcsdict['CRVAL1'], wcsdict['CRVAL2'], crval3]
w.wcs.ctype = [wcsdict['CTYPE1'], wcsdict['CTYPE2'], ctype3]
naxis2 = wcsdict['NAXIS2']
naxis1 = wcsdict['NAXIS1']
else:
w.wcs.crpix = [
templateHeader['CRPIX1'], templateHeader['CRPIX2'], crpix3
]
w.wcs.cdelt = np.array(
[templateHeader['CDELT1'], templateHeader['CDELT2'], cdelt3])
w.wcs.crval = [
templateHeader['CRVAL1'], templateHeader['CRVAL2'], crval3
]
w.wcs.ctype = [
templateHeader['CTYPE1'], templateHeader['CTYPE2'], ctype3
]
naxis2 = templateHeader['NAXIS2']
naxis1 = templateHeader['NAXIS1']
outCube = np.zeros((naxis3, naxis2, naxis1))
outWts = np.zeros((naxis2, naxis1))
xmat, ymat = np.meshgrid(np.arange(naxis1),
np.arange(naxis2),
indexing='ij')
xmat = xmat.reshape(xmat.size)
ymat = ymat.reshape(ymat.size)
xmat = xmat.astype(np.int)
ymat = ymat.astype(np.int)
ctr = 0
for thisfile in filelist:
ctr += 1
s = fits.open(thisfile)
print("Now processing {0}".format(thisfile))
print("This is file {0} of {1}".format(ctr, len(filelist)))
for spectrum in console.ProgressBar((s[1].data)): #pre-processing
specData = spectrum['DATA']
#baseline fit
if doBaseline:
specData = baselineSpectrum(specData,
order=1,
baselineIndex=baselineIndex)
# This part takes the TOPOCENTRIC frequency that is at
# CRPIX1 (i.e., CRVAL1) and calculates the what frequency
# that would have in the LSRK frame with freqShiftValue.
# This then compares to the desired frequency CRVAL3.
DeltaNu = freqShiftValue(spectrum['CRVAL1'],
-spectrum['VFRAME']) - crval3
DeltaChan = DeltaNu / cdelt3
specData = channelShift(specData, -DeltaChan)
outslice = (specData)[startChannel:endChannel]
spectrum_wt = np.isfinite(outslice).astype(np.float)
outslice = np.nan_to_num(outslice)
xpoints, ypoints, zpoints = w.wcs_world2pix(
spectrum['CRVAL2'], spectrum['CRVAL3'], spectrum['CRVAL1'], 0)
tsys = spectrum['TSYS']
if (tsys>10) and (xpoints > 0) and (xpoints < naxis1) \
and (ypoints >0) and (ypoints < naxis2):
pixelWeight, Index = gridFunction(xmat,
ymat,
xpoints,
ypoints,
pixPerBeam=pixPerBeam)
vector = np.outer(outslice * spectrum_wt,
pixelWeight / tsys**2)
vector_wts = np.outer(spectrum_wt, pixelWeight / tsys**2)
wts = pixelWeight / tsys**2
outCube[:, ymat[Index], xmat[Index]] += vector
outWts[ymat[Index], xmat[Index]] += wts
# Temporarily do a file write for every batch of scans.
outWtsTemp = np.copy(outWts)
outWtsTemp.shape = (1, ) + outWtsTemp.shape
outCubeTemp = np.copy(outCube)
outCubeTemp /= outWtsTemp
hdr = fits.Header(w.to_header())
hdr = addHeader_nonStd(hdr, beamSize, Data_Unit)
#
hdu = fits.PrimaryHDU(outCubeTemp, header=hdr)
hdu.writeto(dirname + '.fits', clobber=True)
outWts.shape = (1, ) + outWts.shape
outCube /= outWts
# Create basic fits header from WCS structure
hdr = fits.Header(w.to_header())
# Add non standard fits keyword
hdr = addHeader_nonStd(hdr, beamSize, Data_Unit)
#
hdu = fits.PrimaryHDU(outCube, header=hdr)
hdu.writeto(dirname + '.fits', clobber=True)
w2 = w.dropaxis(2)
hdr2 = fits.Header(w2.to_header())
hdu2 = fits.PrimaryHDU(outWts, header=hdr2)
hdu2.writeto(dirname + '_wts.fits', clobber=True)
def griddata(filelist,
pixPerBeam=3.5,
templateHeader=None,
gridFunction=jincGrid,
startChannel=1024,
endChannel=3072,
doBaseline=True,
baselineRegion=None,
blorder=1,
rebase=None,
rebaseorder=None,
beamSize=None,
OnlineDoppler=True,
flagRMS=False,
flagRipple=False,
flagSpike=False,
blankSpike=False,
plotTimeSeries=False,
rmsThresh=1.25,
spikeThresh=10,
projection='TAN',
plotsubdir='',
outdir=None,
outname=None,
dtype=np.float64,
flagSpatialOutlier=False,
gainDict=None,
**kwargs):
"""Gridding code for GBT spectral scan data produced by pipeline.
Parameters
----------
filelist : list
List of FITS files to be gridded into an output
Keywords
--------
pixPerBeam : float
Number of pixels per beam FWHM
templateHeader : `Header` object
Template header used for spatial pixel grid.
gridFunction : function
Gridding function to be used. The default `jincGrid` is a
tapered circular Bessel function. The function has call
signature of func(xPixelCentre, yPixelCenter, xData, yData,
pixPerBeam)
startChannel : int
Starting channel for spectrum within the original spectral data.
endChannel : int
End channel for spectrum within the original spectral data
doBaseline : bool
Setting to True (default) performs per-scan baseline corrections.
baselineRegion : `numpy.slice` or list of `numpy.slice`
Regions in the original pixel data used for fitting the baseline.
blorder : int
Order of baseline. Defaults to 1 (linear)
rebase : bool
Setting to True (default is False) performs per-pixel
rebaselining of the resulting cube.
beamSize : float
Telescope beam size at this frequency measured in degrees.
OnlineDopper : bool
Setting to True (default) assumes that the Doppler corrections
in the data are corrected during a telescope scan. Setting to
False assumes that the Doppler correction is updated at the
end of a scan and linearly interpolates between scan ends.
flagRMS : bool
Setting to True (default = False) flags spectra with rms
values >rmsThresh x higher than prediction from the radiometer
formula. This rms determination assumes that channels are not
strongly correlated
rmsThresh : float
Threshold for scan flagging based on rms. Default = 1.5
flagRipple : bool
Setting to True (default = False) flags spectra with structure
in the line that is 2x higher than the rms prediction of the
radiometer formula. Note that these estimators are outlier
robust.
flagSpike : bool
Setting to True (default = False) flags regions in spectra
that show jumps of > 5 times the typical pixel to pixel fluctuation.
blankSpike : bool
Setting to True sets spikes to zero to avoid corrupting data
before frequency shifting.
plotTimeSeries : bool
Create scan vs frequency plot to inspect raw scan data. This
saves a PNG file to the output directory.
plotsubdir : str
Subdirectory for timeseries plots. Defaults to same directory as imaging.
outdir : str
Output directory name. Defaults to current working directory.
outname : str
Output directory file name. Defaults to object name in the
original spectra.
flagSpatialOutlier : bool
Setting to True will remove scans with positions far outside the
bounding box of the regular scan pattern. Used to catch instances
where the encoder records erroneous positions.
gainDict : dict
Dictionary that has a tuple of feed and polarization numbers
as keys and returns the gain values for that feed.
Returns
-------
None
"""
eulerFlag = False
print("Starting Gridding")
if outdir is None:
outdir = os.getcwd()
if baselineRegion is None:
baselineRegion = [slice(1024, 1536, 1), slice(2560, 3072, 1)]
if len(filelist) == 0:
warnings.warn('There are no FITS files to process ')
return
# check that every file in the filelist is valid
# If not then remove it and send warning message
for file_i in filelist:
try:
fits.open(file_i)
except:
warnings.warn('file {0} is corrupted'.format(file_i))
filelist.remove(file_i)
# pull a test structure
hdulist = fits.open(filelist[0])
s = hdulist[1].data
# Constants block
sqrt2 = np.sqrt(2)
mad2rms = 1.4826
prefac = mad2rms / sqrt2
c = 299792458.
nu0 = s[0]['RESTFREQ']
Data_Unit = s[0]['TUNIT7']
if outname is None:
outname = s[0]['OBJECT']
# New Beam size measurements use 1.18 vs. 1.22 based on GBT Memo 296.
if beamSize is None:
beamSize = 1.18 * (c / nu0 / 100.0) * 180 / np.pi # in degrees
naxis3 = len(s[0]['DATA'][startChannel:endChannel])
# Default behavior is to park the object velocity at
# the center channel in the VRAD-LSR frame
crval3 = s[0]['RESTFREQ'] * (1 - s[0]['VELOCITY'] / c)
crpix3 = s[0]['CRPIX1'] - startChannel
ctype3 = s[0]['CTYPE1']
cdelt3 = s[0]['CDELT1']
w = wcs.WCS(naxis=3)
w.wcs.restfrq = nu0
# We are forcing this conversion to make nice cubes.
w.wcs.specsys = 'LSRK'
w.wcs.ssysobs = 'TOPOCENT'
if templateHeader is None:
wcsdict = autoHeader(filelist,
beamSize=beamSize,
pixPerBeam=pixPerBeam,
projection=projection)
w.wcs.crpix = [wcsdict['CRPIX1'], wcsdict['CRPIX2'], crpix3]
w.wcs.cdelt = np.array([wcsdict['CDELT1'], wcsdict['CDELT2'], cdelt3])
w.wcs.crval = [wcsdict['CRVAL1'], wcsdict['CRVAL2'], crval3]
w.wcs.ctype = [wcsdict['CTYPE1'], wcsdict['CTYPE2'], ctype3]
naxis2 = wcsdict['NAXIS2']
naxis1 = wcsdict['NAXIS1']
w.wcs.radesys = s[0]['RADESYS']
w.wcs.equinox = s[0]['EQUINOX']
else:
w.wcs.crpix = [
templateHeader['CRPIX1'], templateHeader['CRPIX2'], crpix3
]
w.wcs.cdelt = np.array(
[templateHeader['CDELT1'], templateHeader['CDELT2'], cdelt3])
w.wcs.crval = [
templateHeader['CRVAL1'], templateHeader['CRVAL2'], crval3
]
w.wcs.ctype = [
templateHeader['CTYPE1'], templateHeader['CTYPE2'], ctype3
]
naxis2 = templateHeader['NAXIS2']
naxis1 = templateHeader['NAXIS1']
w.wcs.radesys = templateHeader['RADESYS']
w.wcs.equinox = templateHeader['EQUINOX']
pixPerBeam = np.abs(beamSize / w.pixel_scale_matrix[1, 1])
if pixPerBeam < 3.5:
warnings.warn('Template header requests {0}'.format(pixPerBeam) +
' pixels per beam.')
if (((w.wcs.ctype[0]).split('-'))[0] !=
((s[0]['CTYPE1']).split('-'))[0]):
warnings.warn('Spectral data not in same frame as template header')
eulerFlag = True
outCube = np.zeros((int(naxis3), int(naxis2), int(naxis1)), dtype=dtype)
outWts = np.zeros((int(naxis2), int(naxis1)), dtype=dtype)
xmat, ymat = np.meshgrid(np.arange(naxis1),
np.arange(naxis2),
indexing='ij')
xmat = xmat.reshape(xmat.size)
ymat = ymat.reshape(ymat.size)
xmat = xmat.astype(np.int)
ymat = ymat.astype(np.int)
ctr = 0
for thisfile in filelist:
ctr += 1
s = fits.open(thisfile)
if flagSpatialOutlier:
# Remove outliers in Lat/Lon space
f = np.where(is_outlier(s[1].data['CRVAL2'], thresh=1.5) != True)
s[1].data = s[1].data[f]
f = np.where(is_outlier(s[1].data['CRVAL3'], thresh=1.5) != True)
s[1].data = s[1].data[f]
print("Now processing {0}".format(thisfile))
print("This is file {0} of {1}".format(ctr, len(filelist)))
if len(s[1].data) == 0:
warnings.warn("Corrupted file: {0}".format(thisfile))
continue
nuindex = np.arange(len(s[1].data['DATA'][0]))
if not OnlineDoppler:
vframe = VframeInterpolator(s[1].data)
else:
vframe = s[1].data['VFRAME']
flagct = 0
if eulerFlag:
if 'GLON' in s[1].data['CTYPE2'][0]:
inframe = 'galactic'
elif 'RA' in s[1].data['CTYPE2'][0]:
inframe = 'fk5'
else:
raise NotImplementedError
if 'GLON' in w.wcs.ctype[0]:
outframe = 'galactic'
elif 'RA' in w.wcs.ctype[0]:
outframe = 'fk5'
else:
raise NotImplementedError
coords = SkyCoord(s[1].data['CRVAL2'],
s[1].data['CRVAL3'],
unit=(u.deg, u.deg),
frame=inframe)
coords_xform = coords.transform_to(outframe)
if outframe == 'fk5':
longCoord = coords_xform.ra.deg
latCoord = coords_xform.dec.deg
elif outframe == 'galactic':
longCoord = coords_xform.l.deg
latCoord = coords_xform.b.deg
else:
longCoord = s[1].data['CRVAL2']
latCoord = s[1].data['CRVAL3']
if plotTimeSeries:
vmin = np.nanpercentile(s[1].data['DATA'], 15)
vmed = np.nanpercentile(s[1].data['DATA'], 50)
vmax = np.nanpercentile(s[1].data['DATA'], 85)
fig = plt.figure(figsize=(8.0, 6.5))
ax = fig.add_subplot(111)
im = ax.imshow(s[1].data['DATA'],
interpolation='nearest',
cmap='PuOr',
vmin=(4 * vmin - 3 * vmed),
vmax=4 * vmax - 3 * vmed)
outscans = np.zeros_like(s[1].data['DATA'] + np.nan)
ax.set_xlabel('Channel')
ax.set_title((thisfile.split('/'))[-1])
ax.set_ylabel('Scan')
cb = fig.colorbar(im)
cb.set_label('Intensity (K)')
thisroot = (thisfile.split('/'))[-1]
plt.savefig(outdir + '/' + plotsubdir + '/' +
thisroot.replace('fits', 'png'))
plt.close()
plt.clf()
for idx, spectrum in enumerate(console.ProgressBar((s[1].data))):
# Generate Baseline regions
baselineIndex = np.concatenate(
[nuindex[ss] for ss in baselineRegion])
specData = spectrum['DATA']
if gainDict:
try:
feedwt = 1.0 / gainDict[(str(spectrum['FDNUM']).strip(),
str(spectrum['PLNUM']).strip())]
except KeyError:
continue
else:
feedwt = 1.0
if spectrum['OBJECT'] == 'VANE' or spectrum['OBJECT'] == 'SKY':
continue
# baseline fit
if blankSpike:
jumps = (specData - np.roll(specData, -1))
noise = mad1d(jumps) * 2**(-0.5)
spikemask = (np.abs(jumps) < spikeThresh * noise)
spikemask = spikemask * np.roll(spikemask, 1)
specData[~spikemask] = 0.0
if doBaseline & np.all(np.isfinite(specData)):
specData = baselineSpectrum(specData,
order=blorder,
baselineIndex=baselineIndex)
# This part takes the TOPOCENTRIC frequency that is at
# CRPIX1 (i.e., CRVAL1) and calculates the what frequency
# that would have in the LSRK frame with freqShiftValue.
# This then compares to the desired frequency CRVAL3.
DeltaNu = freqShiftValue(spectrum['CRVAL1'], -vframe[idx]) - crval3
DeltaChan = DeltaNu / cdelt3
specData = channelShift(specData, -DeltaChan)
outslice = (specData)[startChannel:endChannel]
spectrum_wt = np.isfinite(outslice).astype(np.float) * feedwt
outslice = np.nan_to_num(outslice)
xpoints, ypoints, zpoints = w.wcs_world2pix(
longCoord[idx], latCoord[idx], spectrum['CRVAL1'], 0)
tsys = spectrum['TSYS']
if flagRMS:
radiometer_rms = tsys / np.sqrt(
np.abs(spectrum['CDELT1']) * spectrum['EXPOSURE'])
scan_rms = prefac * np.median(
np.abs(outslice[0:-2] - outslice[2:]))
if scan_rms > rmsThresh * radiometer_rms:
tsys = 0 # Blank spectrum
if flagRipple:
scan_rms = prefac * np.median(
np.abs(outslice[0:-2] - outslice[2:]))
ripple = prefac * sqrt2 * np.median(np.abs(outslice))
if ripple > 2 * scan_rms:
tsys = 0 # Blank spectrum
if tsys == 0:
flagct += 1
if (tsys > 10) and (xpoints > 0) and (xpoints < naxis1) \
and (ypoints > 0) and (ypoints < naxis2):
if plotTimeSeries:
outscans[idx, startChannel:endChannel] = outslice
pixelWeight, Index = gridFunction(xmat, ymat, xpoints, ypoints,
pixPerBeam)
vector = np.outer(outslice * spectrum_wt,
pixelWeight / tsys**2)
wts = pixelWeight / tsys**2
outCube[:, ymat[Index], xmat[Index]] += vector
outWts[ymat[Index], xmat[Index]] += wts
print("Percentage of flagged scans: {0:4.2f}".format(100 * flagct /
float(idx)))
if plotTimeSeries:
vmin = np.nanpercentile(outscans, 15)
vmed = np.nanpercentile(outscans, 50)
vmax = np.nanpercentile(outscans, 85)
fig = plt.figure(figsize=(8.0, 6.5))
ax = fig.add_subplot(111)
im = ax.imshow(outscans,
interpolation='nearest',
cmap='PuOr',
vmin=(4 * vmin - 3 * vmed),
vmax=4 * vmax - 3 * vmed)
outscans = np.zeros_like(s[1].data['DATA'] + np.nan)
ax.set_xlabel('Channel')
ax.set_title((thisfile.split('/'))[-1])
ax.set_ylabel('Scan')
cb = fig.colorbar(im)
cb.set_label('Intensity (K)')
thisroot = (thisfile.split('/'))[-1]
plt.savefig(outdir + '/' + plotsubdir + '/' +
thisroot.replace('fits', 'flagged.png'))
plt.close()
plt.clf()
# Temporarily do a file write for every batch of scans.
outWtsTemp = np.copy(outWts)
outWtsTemp.shape = (1, ) + outWtsTemp.shape
outCubeTemp = np.copy(outCube)
outCubeTemp /= outWtsTemp
hdr = fits.Header(w.to_header())
hdr = addHeader_nonStd(hdr, beamSize, s[1].data)
#
hdu = fits.PrimaryHDU(outCubeTemp, header=hdr)
hdu.writeto(outdir + '/' + outname + '.fits', overwrite=True)
outWts.shape = (1, ) + outWts.shape
outCube /= outWts
# Create basic fits header from WCS structure
hdr = fits.Header(w.to_header())
# Add non standard fits keyword
hdr = addHeader_nonStd(hdr, beamSize, s[1].data[0])
# Adds history message
# try:
# hdr.add_history(history_message)
# except UnboundLocalError:
# pass
hdr.add_history('Using GBTPIPE gridder version {0}'.format(__version__))
hdu = fits.PrimaryHDU(outCube, header=hdr)
hdu.writeto(outdir + '/' + outname + '.fits', overwrite=True)
w2 = w.dropaxis(2)
hdr2 = fits.Header(w2.to_header())
hdu2 = fits.PrimaryHDU(outWts, header=hdr2)
hdu2.writeto(outdir + '/' + outname + '_wts.fits', overwrite=True)
if rebase:
if rebaseorder is None:
rebaseorder = blorder
if 'NH3_11' in outname:
Baseline.rebaseline(outdir + '/' + outname + '.fits',
windowFunction=Baseline.ammoniaWindow,
line='oneone',
blorder=rebaseorder,
**kwargs)
elif 'NH3_22' in outname:
Baseline.rebaseline(outdir + '/' + outname + '.fits',
windowFunction=Baseline.ammoniaWindow,
line='twotwo',
blorder=rebaseorder,
**kwargs)
elif 'NH3_33' in outname:
Baseline.rebaseline(outdir + '/' + outname + '.fits',
winfunc=Baseline.ammoniaWindow,
blorder=rebaseorder,
line='threethree',
**kwargs)
else:
Baseline.rebaseline(outdir + '/' + outname + '.fits',
blorder=rebaseorder,
windowFunction=Baseline.tightWindow,
**kwargs)
def cygriddata(filelist,
cacheSpectra=False,
pixPerBeam=3.5,
templateHeader=None,
gridFunction=gg.jincGrid,
startChannel=1024,
endChannel=3072,
doBaseline=True,
baselineRegion=None,
blorder=1,
rebase=None,
rebaseorder=None,
beamSize=None,
OnlineDoppler=True,
flagRMS=False,
flagRipple=False,
flagSpike=False,
rmsThresh=1.25,
spikeThresh=10,
projection='TAN',
outdir=None,
outname=None,
dtype='float64',
**kwargs):
"""Gridding code for GBT spectral scan data produced by pipeline using CyGrid
Parameters
----------
filelist : list
List of FITS files to be gridded into an output
Keywords
--------
pixPerBeam : float
Number of pixels per beam FWHM
templateHeader : `Header` object
Template header used for spatial pixel grid.
gridFunction : function
Gridding function to be used. The default `jincGrid` is a
tapered circular Bessel function. The function has call
signature of func(xPixelCentre, yPixelCenter, xData, yData,
pixPerBeam)
startChannel : int
Starting channel for spectrum within the original spectral data.
endChannel : int
End channel for spectrum within the original spectral data
doBaseline : bool
Setting to True (default) performs per-scan baseline corrections.
baselineRegion : `numpy.slice` or list of `numpy.slice`
Regions in the original pixel data used for fitting the baseline.
blorder : int
Order of baseline. Defaults to 1 (linear)
rebase : bool
Setting to True (default is False) performs per-pixel
rebaselining of the resulting cube.
beamSize : float
Telescope beam size at this frequency measured in degrees.
OnlineDopper : bool
Setting to True (default) assumes that the Doppler corrections
in the data are corrected during a telescope scan. Setting to
False assumes that the Doppler correction is updated at the
end of a scan and linearly interpolates between scan ends.
flagRMS : bool
Setting to True (default = False) flags spectra with rms
values >rmsThresh x higher than prediction from the radiometer
formula. This rms determination assumes that channels are not
strongly correlated
rmsThresh : float
Threshold for scan flagging based on rms. Default = 1.5
flagRipple : bool
Setting to True (default = False) flags spectra with structure
in the line that is 2x higher than the rms prediction of the
radiometer formula. Note that these estimators are outlier
robust.
flagSpike : bool
Setting to True (default = False) flags regions in spectra
that show jumps of > 5 times the typical pixel to pixel fluctuation.
outdir : str
Output directory name. Defaults to current working directory.
outname : str
Output directory file name. Defaults to object name in the
original spectra.
Returns
-------
None
"""
if outdir is None:
outdir = os.getcwd()
if baselineRegion is None:
baselineRegion = [slice(1024, 1536, 1), slice(2560, 3072, 1)]
if len(filelist) == 0:
warnings.warn('There are no FITS files to process ')
return
# check that every file in the filelist is valid
# If not then remove it and send warning message
for file_i in filelist:
try:
fits.open(file_i)
except:
warnings.warn('file {0} is corrupted'.format(file_i))
filelist.remove(file_i)
# pull a test structure
hdulist = fits.open(filelist[0])
s = hdulist[1].data
# Constants block
sqrt2 = np.sqrt(2)
mad2rms = 1.4826
prefac = mad2rms / sqrt2
c = 299792458.
nu0 = s[0]['RESTFREQ']
Data_Unit = s[0]['TUNIT7']
if outname is None:
outname = s[0]['OBJECT']
# New Beam size measurements use 1.18 vs. 1.22 based on GBT Memo 296.
if beamSize is None:
beamSize = 1.18 * (c / nu0 / 100.0) * 180 / np.pi # in degrees
naxis3 = len(s[0]['DATA'][startChannel:endChannel])
# Default behavior is to park the object velocity at
# the center channel in the VRAD-LSR frame
crval3 = s[0]['RESTFREQ'] * (1 - s[0]['VELOCITY'] / c)
crpix3 = s[0]['CRPIX1'] - startChannel
ctype3 = s[0]['CTYPE1']
cdelt3 = s[0]['CDELT1']
w = wcs.WCS(naxis=3)
w.wcs.restfrq = nu0
# We are forcing this conversion to make nice cubes.
w.wcs.specsys = 'LSRK'
w.wcs.ssysobs = 'TOPOCENT'
if templateHeader is None:
wcsdict = gg.autoHeader(filelist,
beamSize=beamSize,
pixPerBeam=pixPerBeam,
projection=projection)
w.wcs.crpix = [wcsdict['CRPIX1'], wcsdict['CRPIX2'], crpix3]
w.wcs.cdelt = np.array([wcsdict['CDELT1'], wcsdict['CDELT2'], cdelt3])
w.wcs.crval = [wcsdict['CRVAL1'], wcsdict['CRVAL2'], crval3]
w.wcs.ctype = [wcsdict['CTYPE1'], wcsdict['CTYPE2'], ctype3]
naxis2 = wcsdict['NAXIS2']
naxis1 = wcsdict['NAXIS1']
w.wcs.radesys = s[0]['RADESYS']
w.wcs.equinox = s[0]['EQUINOX']
else:
w.wcs.crpix = [
templateHeader['CRPIX1'], templateHeader['CRPIX2'], crpix3
]
w.wcs.cdelt = np.array(
[templateHeader['CDELT1'], templateHeader['CDELT2'], cdelt3])
w.wcs.crval = [
templateHeader['CRVAL1'], templateHeader['CRVAL2'], crval3
]
w.wcs.ctype = [
templateHeader['CTYPE1'], templateHeader['CTYPE2'], ctype3
]
naxis2 = templateHeader['NAXIS2']
naxis1 = templateHeader['NAXIS1']
w.wcs.radesys = templateHeader['RADESYS']
w.wcs.equinox = templateHeader['EQUINOX']
pixPerBeam = np.abs(beamSize / w.pixel_scale_matrix[1, 1])
if pixPerBeam < 3.5:
warnings.warn('Template header requests {0}'.format(pixPerBeam) +
' pixels per beam.')
if (((w.wcs.ctype[0]).split('-'))[0] !=
((s[0]['CTYPE1']).split('-'))[0]):
warnings.warn('Spectral data not in same frame as template header')
eulerFlag = True
# outCube = np.zeros((int(naxis3), int(naxis2), int(naxis1)),dtype=dtype)
# outWts = np.zeros((int(naxis2), int(naxis1)),dtype=dtype)
xmat, ymat = np.meshgrid(np.arange(naxis1),
np.arange(naxis2),
indexing='ij')
xmat = xmat.reshape(xmat.size)
ymat = ymat.reshape(ymat.size)
xmat = xmat.astype(np.int)
ymat = ymat.astype(np.int)
ctr = 0
speclist = []
lonlist = []
latlist = []
for thisfile in filelist:
ctr += 1
cachefile = thisfile.replace('.fits', '_speccache.npz')
if cacheSpectra and os.path.isfile(cachefile):
npzfile = np.load(cachefile)
speclist += npzfile['speclist'].tolist()
lonlist += npzfile['lonlist'].tolist()
latlist += npzfile['latlist'].tolist()
else:
s = fits.open(thisfile)
print("Now processing {0}".format(thisfile))
print("This is file {0} of {1}".format(ctr, len(filelist)))
nuindex = np.arange(len(s[1].data['DATA'][0]))
if not OnlineDoppler:
vframe = gg.VframeInterpolator(s[1].data)
else:
vframe = s[1].data['VFRAME']
flagct = 0
if eulerFlag:
if 'GLON' in s[1].data['CTYPE2'][0]:
inframe = 'galactic'
elif 'RA' in s[1].data['CTYPE2'][0]:
inframe = 'fk5'
else:
raise NotImplementedError
if 'GLON' in w.wcs.ctype[0]:
outframe = 'galactic'
elif 'RA' in w.wcs.ctype[0]:
outframe = 'fk5'
else:
raise NotImplementedError
coords = SkyCoord(s[1].data['CRVAL2'],
s[1].data['CRVAL3'],
unit=(u.deg, u.deg),
frame=inframe)
coords_xform = coords.transform_to(outframe)
if outframe == 'fk5':
longCoord = coords_xform.ra.deg
latCoord = coords_xform.dec.deg
elif outframe == 'galactic':
longCoord = coords_xform.l.deg
latCoord = coords_xform.b.deg
else:
longCoord = s[1].data['CRVAL2'],
latCoord = s[1].data['CRVAL3']
lonlist += [longCoord]
latlist += [latCoord]
for idx, spectrum in enumerate(console.ProgressBar((s[1].data))):
# Generate Baseline regions
baselineIndex = np.concatenate(
[nuindex[ss] for ss in baselineRegion])
specData = spectrum['DATA']
# baseline fit
if doBaseline & np.all(np.isfinite(specData)):
specData = gg.baselineSpectrum(specData,
order=blorder,
baselineIndex=baselineIndex)
# This part takes the TOPOCENTRIC frequency that is at
# CRPIX1 (i.e., CRVAL1) and calculates the what frequency
# that would have in the LSRK frame with freqShiftValue.
# This then compares to the desired frequency CRVAL3.
DeltaNu = gg.freqShiftValue(spectrum['CRVAL1'],
-vframe[idx]) - crval3
DeltaChan = DeltaNu / cdelt3
specData = gg.channelShift(specData, -DeltaChan)
outslice = (specData)[startChannel:endChannel]
spectrum_wt = np.isfinite(outslice).astype(np.float)
outslice = np.nan_to_num(outslice)
# xpoints, ypoints, zpoints = w.wcs_world2pix(longCoord[idx],
# latCoord[idx],
# spectrum['CRVAL1'], 0)
tsys = spectrum['TSYS']
if flagSpike:
jumps = (outslice - np.roll(outslice, -1))
noise = gg.mad1d(jumps) * 2**(-0.5)
spikemask = (np.abs(jumps) < spikeThresh * noise)
spikemask = spikemask * np.roll(spikemask, 1)
spectrum_wt *= spikemask
if flagRMS:
radiometer_rms = tsys / np.sqrt(
np.abs(spectrum['CDELT1']) * spectrum['EXPOSURE'])
scan_rms = prefac * np.median(
np.abs(outslice[0:-2] - outslice[2:]))
if scan_rms > rmsThresh * radiometer_rms:
tsys = 0 # Blank spectrum
if flagRipple:
scan_rms = prefac * np.median(
np.abs(outslice[0:-2] - outslice[2:]))
ripple = prefac * sqrt2 * np.median(np.abs(outslice))
if ripple > 2 * scan_rms:
tsys = 0 # Blank spectrum
if tsys == 0:
flagct += 1
speclist += [outslice]
# if (tsys > 10) and (xpoints > 0) and (xpoints < naxis1) \
# and (ypoints > 0) and (ypoints < naxis2):
# pixelWeight, Index = gridFunction(xmat, ymat,
# xpoints, ypoints,
# pixPerBeam)
# vector = np.outer(outslice * spectrum_wt,
# pixelWeight / tsys**2)
# wts = pixelWeight / tsys**2
# outCube[:, ymat[Index], xmat[Index]] += vector
# outWts[ymat[Index], xmat[Index]] += wts
print("Percentage of flagged scans: {0:4.2f}".format(100 * flagct /
float(idx)))
# Temporarily do a file write for every batch of scans.
if cacheSpectra:
np.savez(thisfile.replace('.fits', '_speccache.npz'),
lonlist=lonlist,
latlist=latlist,
speclist=speclist)
try:
lonlist = np.squeeze(np.stack(lonlist))
latlist = np.squeeze(np.stack(latlist))
speclist = np.squeeze(np.stack(speclist))
except ValueError:
lonlist = np.squeeze(np.hstack(lonlist))
latlist = np.squeeze(np.hstack(latlist))
speclist = np.squeeze(np.stack(speclist))
# DO THE GRIDDING
hdr = w.to_header()
hdr['NAXIS1'] = naxis1
hdr['NAXIS2'] = naxis2
hdr['NAXIS3'] = 10
gridder = cygrid.WcsGrid(hdr)
gridder.set_kernel('gauss1d', (beamSize / 2.355 / 2),
2 * (beamSize / 2.355), beamSize / 2 / 2.355)
gridder.grid(lonlist, latlist, speclist, dtype=dtype)
hdr = fits.Header(w.to_header())
s = fits.open(thisfile)
hdr = gg.addHeader_nonStd(hdr, beamSize, s[1].data[0])
hdr.add_history('Using GBTPIPE gridder version {0}'.format(__version__))
hdu = fits.PrimaryHDU(gridder.get_datacube(), header=hdr)
hdu.writeto(outdir + '/' + outname + '.fits', clobber=True)
# outWts.shape = (1,) + outWts.shape
# outCube /= outWts
# hdu = fits.PrimaryHDU(outCube, header=hdr)
# w2 = w.dropaxis(2)
# hdr2 = fits.Header(w2.to_header())
# hdu2 = fits.PrimaryHDU(outWts, header=hdr2)
# hdu2.writeto(outdir + '/' + outname + '_wts.fits', clobber=True)
if rebase:
if rebaseorder is None:
rebaseorder = blorder
if 'NH3_11' in outname:
Baseline.rebaseline(outdir + '/' + outname + '.fits',
windowFunction=Baseline.ammoniaWindow,
line='oneone',
blorder=rebaseorder,
**kwargs)
elif 'NH3_22' in outname:
Baseline.rebaseline(outdir + '/' + outname + '.fits',
windowFunction=Baseline.ammoniaWindow,
line='twotwo',
blorder=rebaseorder,
**kwargs)
elif 'NH3_33' in outname:
Baseline.rebaseline(outdir + '/' + outname + '.fits',
winfunc=Baseline.ammoniaWindow,
blorder=rebaseorder,
line='threethree',
**kwargs)
else:
Baseline.rebaseline(outdir + '/' + outname + '.fits',
blorder=rebaseorder,
windowFunction=Baseline.tightWindow,
**kwargs)
def residual_cube(cubename, fitfile=None,
expand=20, writemodel=False, fileprefix=None,
filesuffix=None,
writeresidual=False, writechisq=True):
"""This function generates products for evaluating the goodness of
fit for a cold_ammonia model. Either a parameter cube name must
be passed or the prefix/suffixes of the files.
Parameters
----------
cubename : str
Name of the original data file
Keywords
--------
fitfile : str
Name of the parameter file produced by the cube fitter
fileprefix : str
Prefix of file name before the parameter name (e.g., for
L1688_N_NH3_DR1_rebase3_flag.fits, this would be 'L1688_').
filesufffix : str
Prefix of file name before the parameter name (e.g., for
L1688_N_NH3_DR1_rebase3_flag.fits, this would be
'_DR1_rebase3_flag').
expand : int
Expands the region where the residual is evaluated by this
many channels in the spectral dimension
writemodel : bool
Setting to True writes out a model cube of the ammonia fit
writeresidual : bool
Setting to True writes out a residual cube
writechisq : bool
Setting to True writes out a map of the reduced chi-squared
goodness of fit.
Returns
-------
None
"""
try:
if fitfile is None:
tkin = fits.getdata(fileprefix + 'Tkin' + filesuffix + '.fits')
tex = fits.getdata(fileprefix + 'Tex' + filesuffix + '.fits')
sigma = fits.getdata(fileprefix + 'Sigma' + filesuffix + '.fits')
column = fits.getdata(fileprefix + 'N_NH3' + filesuffix + '.fits')
v0 = fits.getdata(fileprefix + 'Vlsr' + filesuffix + '.fits')
fortho = np.zeros_like(v0)
else:
hdu = fits.open(fitfile)
fitparams = hdu[0].data
tkin = fitparams[0, :, :]
tex = fitparams[1, :, :]
column = fitparams[2, :, :]
sigma = fitparams[3, :, :]
v0 = fitparams[4, :, :]
fortho = fitparams[5, :,:]
except NameError:
warnings.warn("Either fitfile or fileprefix/filesuffix"+
" must be specified")
cube = SpectralCube.read(cubename)
cube = cube.with_spectral_unit(u.km/u.s,velocity_convention='radio')
model = np.zeros(cube.shape)
yy, xx = np.where(column>0)
cube = cube.with_spectral_unit(u.Hz)
spaxis = pyspeckit.spectrum.units.SpectroscopicAxis(cube.spectral_axis)
for y, x in console.ProgressBar(zip(yy,xx)):
fit = ammonia.cold_ammonia(spaxis, tkin[y, x], tex=tex[y, x],
ntot=column[y, x], width=sigma[y, x],
xoff_v=v0[y, x], fortho=fortho[y, x])
model[:, y, x] = fit
mask = model > 0
residual = cube.filled_data[:].value-model
# This calculates chisq over the region where the fit is non-zero
# plus a buffer of size set by the expand keyword.
selem = np.ones(expand,dtype=np.bool)
selem.shape += (1,1,)
mask = nd.binary_dilation(mask, selem)
mask = mask.astype(np.float)
chisq = np.sum((residual * mask)**2, axis=0) / np.sum(mask, axis=0)
# This produces a robust estimate of the RMS along every line of sight:
diff = residual - np.roll(residual, 2, axis=0)
rms = 1.4826 * np.nanmedian(np.abs(diff), axis=0) / 2**0.5
chisq /= rms**2
root = (cubename.split('.'))[0]
if writechisq:
hdu = fits.PrimaryHDU(chisq, cube.wcs.celestial.to_header())
hdu.writeto(root+'_chisq.fits', clobber=True)
if writeresidual:
newcube = SpectralCube(residual,cube.wcs,header=cube.header)
newcube.write(root+'_residual.fits', overwrite=True)
if writemodel:
model = SpectralCube(model, cube.wcs, header=cube.header)
model.write(root + '_model.fits', overwrite=True)
