def plotFluxRatiosDist(predFlux, measFlux, RA, Dec, refRA, refDec, labels,
outDir, name1, name2, format, clip=True):
"""
Makes plot of measured-to-predicted flux ratio vs. distance from center
"""
import numpy as np
from ..operations_lib import calculateSeparation
try:
from astropy.stats.funcs import sigma_clip
except ImportError:
from astropy.stats import sigma_clip
try:
import matplotlib
if matplotlib.get_backend() is not 'Agg':
matplotlib.use("Agg")
import matplotlib.pyplot as plt
from matplotlib.ticker import FuncFormatter
except Exception as e:
raise ImportError('PyPlot could not be imported. Plotting is not '
'available: {0}'.format(e.message))
if name1 is None:
name1 = 'Model 1'
if name2 is None:
name2 = 'Model 2'
ratio = measFlux / predFlux
separation = np.zeros(len(measFlux))
for i in range(len(measFlux)):
separation[i] = calculateSeparation(RA[i], Dec[i], refRA, refDec).value
fig = plt.figure(figsize=(7.0, 5.0))
ax1 = plt.subplot(1, 1, 1)
ax1.plot(separation, ratio, 'o')
plt.title('Flux Density Ratios ({0} / {1})'.format(name1, name2))
plt.ylabel('Flux density ratio')
plt.xlabel('Distance from center (deg)')
# Calculate mean ratio and std dev.
if clip:
mean_ratio = np.mean(sigma_clip(ratio))
std = np.std(sigma_clip(ratio))
else:
mean_ratio = np.mean(ratio)
std = np.std(ratio)
ax1.set_ylim(0, 2.0*mean_ratio)
xmin, xmax, ymin, ymax = plt.axis()
ax1.plot([0.0, xmax], [mean_ratio, mean_ratio], '--g')
ax1.plot([0.0, xmax], [mean_ratio+std, mean_ratio+std], '-.g')
ax1.plot([0.0, xmax], [mean_ratio-std, mean_ratio-std], '-.g')
if labels is not None:
xls = separation
yls = ratio
for label, xl, yl in zip(labels, xls, yls):
plt.annotate(label, xy = (xl, yl), xytext = (-2, 2), textcoords=
'offset points', ha='right', va='bottom')
plt.savefig(outDir+'flux_ratio_vs_distance.{}'.format(format), format=format)
def plotFluxRatiosFlux(predFlux, measFlux, labels, outDir, clip=True):
"""
Makes plot of measured-to-predicted flux ratio vs. flux
"""
import os
import numpy as np
from ..operations_lib import calculateSeparation
try:
from astropy.stats.funcs import sigma_clip
except ImportError:
from astropy.stats import sigma_clip
try:
import matplotlib
if matplotlib.get_backend() is not 'Agg':
matplotlib.use("Agg")
import matplotlib.pyplot as plt
from matplotlib.ticker import FuncFormatter
except Exception as e:
raise ImportError('PyPlot could not be imported. Plotting is not '
'available: {0}'.format(e.message))
ratio = measFlux / predFlux
fig = plt.figure(figsize=(7.0, 5.0))
ax1 = plt.subplot(1, 1, 1)
ax1.plot(measFlux, ratio, 'o')
ax1.set_ylim(0, 2)
plt.title('Flux Ratios (Model 1 / Model 2)')
plt.ylabel('Flux ratio')
plt.xlabel('Model 1 flux (Jy)')
# Calculate mean ratio and std dev.
if clip:
mean_ratio = np.mean(sigma_clip(ratio))
std = np.std(sigma_clip(ratio))
else:
mean_ratio = np.mean(ratio)
std = np.std(ratio)
xmin, xmax, ymin, ymax = plt.axis()
ax1.plot([0.0, xmax], [mean_ratio, mean_ratio], '--g')
ax1.plot([0.0, xmax], [mean_ratio + std, mean_ratio + std], '-.g')
ax1.plot([0.0, xmax], [mean_ratio - std, mean_ratio - std], '-.g')
if labels is not None:
xls = measFlux
yls = ratio
for label, xl, yl in zip(labels, xls, yls):
plt.annotate(label,
xy=(xl, yl),
xytext=(-2, 2),
textcoords='offset points',
ha='right',
va='bottom')
plt.savefig(outDir + 'flux_ratio_vs_flux.pdf', format='pdf')
def plotFluxRatiosDist(predFlux, measFlux, RA, Dec, refRA, refDec, labels, outDir, clip=True):
"""
Makes plot of measured-to-predicted flux ratio vs. distance from center
"""
import numpy as np
from ..operations_lib import calculateSeparation
try:
from astropy.stats.funcs import sigma_clip
except ImportError:
from astropy.stats import sigma_clip
try:
import matplotlib
if matplotlib.get_backend() is not "Agg":
matplotlib.use("Agg")
import matplotlib.pyplot as plt
from matplotlib.ticker import FuncFormatter
except Exception as e:
raise ImportError("PyPlot could not be imported. Plotting is not " "available: {0}".format(e.message))
ratio = measFlux / predFlux
separation = np.zeros(len(measFlux))
for i in range(len(measFlux)):
separation[i] = calculateSeparation(RA[i], Dec[i], refRA, refDec).value
fig = plt.figure(figsize=(7.0, 5.0))
ax1 = plt.subplot(1, 1, 1)
ax1.plot(separation, ratio, "o")
ax1.set_ylim(0, 2)
plt.title("Flux Ratios (Model 1 / Model 2)")
plt.ylabel("Flux ratio")
plt.xlabel("Distance from center (deg)")
# Calculate mean ratio and std dev.
if clip:
mean_ratio = np.mean(sigma_clip(ratio))
std = np.std(sigma_clip(ratio))
else:
mean_ratio = np.mean(ratio)
std = np.std(ratio)
xmin, xmax, ymin, ymax = plt.axis()
ax1.plot([0.0, xmax], [mean_ratio, mean_ratio], "--g")
ax1.plot([0.0, xmax], [mean_ratio + std, mean_ratio + std], "-.g")
ax1.plot([0.0, xmax], [mean_ratio - std, mean_ratio - std], "-.g")
if labels is not None:
xls = separation
yls = ratio
for label, xl, yl in zip(labels, xls, yls):
plt.annotate(label, xy=(xl, yl), xytext=(-2, 2), textcoords="offset points", ha="right", va="bottom")
plt.savefig(outDir + "flux_ratio_vs_distance.pdf", format="pdf")
def plotFluxRatiosFlux(predFlux, measFlux, labels, outDir, clip=True):
"""
Makes plot of measured-to-predicted flux ratio vs. flux
"""
import os
import numpy as np
from ..operations_lib import calculateSeparation
try:
from astropy.stats.funcs import sigma_clip
except ImportError:
from astropy.stats import sigma_clip
try:
import matplotlib
if matplotlib.get_backend() is not 'Agg':
matplotlib.use("Agg")
import matplotlib.pyplot as plt
from matplotlib.ticker import FuncFormatter
except Exception as e:
raise ImportError('PyPlot could not be imported. Plotting is not '
'available: {0}'.format(e.message))
ratio = measFlux / predFlux
fig = plt.figure(figsize=(7.0, 5.0))
ax1 = plt.subplot(1, 1, 1)
ax1.plot(measFlux, ratio, 'o')
ax1.set_ylim(0, 2)
plt.title('Flux Ratios (Model 1 / Model 2)')
plt.ylabel('Flux ratio')
plt.xlabel('Model 1 flux (Jy)')
# Calculate mean ratio and std dev.
if clip:
mean_ratio = np.mean(sigma_clip(ratio))
std = np.std(sigma_clip(ratio))
else:
mean_ratio = np.mean(ratio)
std = np.std(ratio)
xmin, xmax, ymin, ymax = plt.axis()
ax1.plot([0.0, xmax], [mean_ratio, mean_ratio], '--g')
ax1.plot([0.0, xmax], [mean_ratio+std, mean_ratio+std], '-.g')
ax1.plot([0.0, xmax], [mean_ratio-std, mean_ratio-std], '-.g')
if labels is not None:
xls = measFlux
yls = ratio
for label, xl, yl in zip(labels, xls, yls):
plt.annotate(label, xy = (xl, yl), xytext = (-2, 2), textcoords=
'offset points', ha='right', va='bottom')
plt.savefig(outDir+'flux_ratio_vs_flux.pdf', format='pdf')
def calculateWeights(self):
"""
finds illum cal factors by making an image per wavelength bin, then smoothing it and dividing the mean of the smoothedImage over pixels by the smoothedImage
"""
cubeWeightsList = []
self.averageSpectra = []
deltaWeightsList = []
self.totalCube = np.sum(self.spectralCubes,axis=0) #sum all cubes
self.totalFrame = np.sum(self.totalCube,axis=-1)#sum over wvl
weights = []
for iWvl in xrange(self.nWvlBins):
wvlSlice = self.totalCube[:,:,iWvl]
wvlSlice[wvlSlice == 0] = np.nan
nanMask = np.isnan(wvlSlice)
#do a sigma-clipping on the wvlSlice and insert it back in the cube
maskedWvlSlice = np.ma.array(wvlSlice,mask=nanMask)
clippedWvlSlice = sigma_clip(wvlSlice,sig=self.nSigmaClip,iters=None,cenfunc=np.ma.median)
wvlSlice[clippedWvlSlice.mask] = np.nan
self.totalCube[:,:,iWvl] = wvlSlice
#do a smoothing over the slice
smoothedWvlSlice = nearestNRobustMeanFilter(wvlSlice,n=self.nNearest,nSigmaClip=self.nSigmaClip)
wvlIllumWeights = np.mean(smoothedWvlSlice)/smoothedWvlSlice
weights.append(wvlIllumWeights)
self.weights = np.array(weights)
#move the wvl dimension to the end
self.weights = np.swapaxes(self.weights,0,1)
self.weights = np.swapaxes(self.weights,1,2)
self.deltaWeights = np.zeros_like(self.weights)
self.flags = np.zeros_like(self.weights)
def sigmas(files, nofiles = 48, plot = False):
'''Returns a mean spectrum in the form of a 2d array - one dimension
is the wavelengths, the other is the flux files.
The input is the start of the filenames; i.e. if you have 1040 data
files named "mortenisthebest00001.txt", "mortenisthebest00002.txt" etc.,
the input is files="mortenisthebest0" (the file format is assumed to
be .txt) and nofiles = 1040.
'''
all_data = np.zeros((nofiles, 2048, 2))
# First loop over the files to get data from each and put the data
# in all_data:
for indx in range(nofiles):
file1 = files + str(indx).zfill(len(str(nofiles))) + '.txt'
data1 = np.genfromtxt(file1, skip_header = 17, skip_footer = 1)
all_data[indx] = data1
datamean = np.zeros((2,2048))
# We want to have a mean array datamean that gives us the x-values
# and mean flux values for each wavelength in the spectrum.
# The x-values are assumed to be the same for every spectrum.
datamean[0] = all_data[0,:,0]
# Do a sigma clip to take away outliers
for i in range(len(all_data[0,:,0])):
bool_clip = sigma_clip(all_data[:,i,1], sig = 1.5)[1]
datamean[1,i] = np.mean(all_data[:,i,1][bool_clip])
# datamean[1] = np.mean(all_data[:,:,1], axis = 0)
if plot == True:
plt.clf()
plt.plot(datamean[0], datamean[1])
plt.show()
plt.semilogy()
plt.axis([530,680,15,60000])
plt.xlabel('Wavelength (nm)', fontsize = 20)
plt.ylabel('Flux (counts)', fontsize = 20)
plt.title('The raw mean of the spectra', fontsize = 25)
return datamean
def _compute_diff(arg):
"""Worker: Computes the DC offset of frame-coadd"""
upperKey, upperPath, upperWeightPath, lowerKey, lowerPath, \
lowerWeightPath, overlap = arg
# print "diff between", upperKey, lowerKey
upper = SliceableImage.makeFromFITS(upperKey, upperPath, upperWeightPath)
upper.setRange(overlap.upSlice)
lower = SliceableImage.makeFromFITS(lowerKey, lowerPath, lowerWeightPath)
lower.setRange(overlap.loSlice)
goodPix = np.where((upper.weight > 0.) & (lower.weight > 0.))
nPixels = len(goodPix[0])
if nPixels > 10:
diff_pixels = upper.image[goodPix] - lower.image[goodPix]
diff_pixels = diff_pixels[np.isfinite(diff_pixels)]
clipped = sigma_clip(diff_pixels, sig=5., iters=1,
varfunc=np.nanvar)
median = np.nanmedian(clipped[~clipped.mask])
sigma = np.nanstd(clipped[~clipped.mask])
diffData = {"diffimage_mean": median,
"diffimage_sigma": sigma,
"area": nPixels}
else:
diffData = None
return upperKey, lowerKey, diffData
illumImg = imgDict['illumImg']
exptime = imgDict['exptime']
nRows,nCols = np.shape(flatImg)
nanMask = np.isnan(rawImg) | np.isnan(flatImg) | np.isnan(illumImg)
rawImg = np.ma.array(rawImg,mask=nanMask)
flatImg = np.ma.array(flatImg,mask=nanMask)
illumImg = np.ma.array(illumImg,mask=nanMask)
bClipData = True
sig=3.
if bClipData:
iters=None
cenfunc = np.ma.median
rawImg = sigma_clip(rawImg,sig=sig,iters=iters,cenfunc=cenfunc)
flatImg = sigma_clip(flatImg,sig=sig,iters=iters,cenfunc=cenfunc)
illumImg = sigma_clip(illumImg,sig=sig,iters=iters,cenfunc=cenfunc)
clipMask = rawImg.mask | flatImg.mask | illumImg.mask
rawImg.mask = clipMask
flatImg.mask = clipMask
illumImg.mask = clipMask
print 'removed ',np.sum(clipMask)-np.sum(nanMask),'pixels'
rawList = rawImg[~rawImg.mask]
flatList = flatImg[~flatImg.mask]
illumList = illumImg[~illumImg.mask]
#plotArray(title='with flatcal',image=flatImg)
def findStats(predFlux, measFlux, RA, Dec, refRA, refDec, outDir, info0, info1,
info2, name1, name2):
"""
Calculates statistics and saves them to 'stats.txt'
"""
import os
import numpy as np
from ..operations_lib import calculateSeparation
try:
from astropy.stats.funcs import sigma_clip
except ImportError:
from astropy.stats import sigma_clip
ratio = measFlux / predFlux
meanRatio = np.mean(ratio)
stdRatio = np.std(ratio)
clippedRatio = sigma_clip(ratio)
meanClippedRatio = np.mean(clippedRatio)
stdClippedRatio = np.std(clippedRatio)
RAOffsets = np.zeros(len(RA))
DecOffsets = np.zeros(len(Dec))
for i in range(len(RA)):
if RA[i] >= refRA[i]:
sign = 1.0
else:
sign = -1.0
RAOffsets[i] = sign * calculateSeparation(RA[i], Dec[i], refRA[i], Dec[i]).value # deg
if Dec[i] >= refDec[i]:
sign = 1.0
else:
sign = -1.0
DecOffsets[i] = sign * calculateSeparation(RA[i], Dec[i], RA[i], refDec[i]).value # deg
meanRAOffset = np.mean(RAOffsets)
stdRAOffset = np.std(RAOffsets)
clippedRAOffsets = sigma_clip(RAOffsets)
meanClippedRAOffset = np.mean(clippedRAOffsets)
stdClippedRAOffset = np.std(clippedRAOffsets)
meanDecOffset = np.mean(DecOffsets)
stdDecOffset = np.std(DecOffsets)
clippedDecOffsets = sigma_clip(DecOffsets)
meanClippedDecOffset = np.mean(clippedDecOffsets)
stdClippedDecOffset = np.std(clippedDecOffsets)
stats = {'meanRatio':meanRatio,
'stdRatio':stdRatio,
'meanClippedRatio':meanClippedRatio,
'stdClippedRatio':stdClippedRatio,
'meanRAOffsetDeg':meanRAOffset,
'stdRAOffsetDeg':stdRAOffset,
'meanClippedRAOffsetDeg':meanClippedRAOffset,
'stdClippedRAOffsetDeg':stdClippedRAOffset,
'meanDecOffsetDeg':meanDecOffset,
'stdDecOffsetDeg':stdDecOffset,
'meanClippedDecOffsetDeg':meanClippedDecOffset,
'stdClippedDecOffsetDeg':stdClippedDecOffset}
outLines = ['Statistics from sky model comparison\n']
outLines.append('------------------------------------\n\n')
outLines.append('Sky model 1 ({}):\n'.format(name1))
outLines.append(info1+'\n\n')
outLines.append('Sky model 2 ({}):\n'.format(name2))
outLines.append(info2+'\n\n')
outLines.append(info0+'\n')
outLines.append('Number of matches found for comparison: {0}\n\n'.format(len(predFlux)))
outLines.append('Mean flux density ratio (1 / 2): {0}\n'.format(meanRatio))
outLines.append('Std. dev. flux density ratio (1 / 2): {0}\n'.format(stdRatio))
outLines.append('Mean 3-sigma-clipped flux density ratio (1 / 2): {0}\n'.format(meanClippedRatio))
outLines.append('Std. dev. 3-sigma-clipped flux density ratio (1 / 2): {0}\n\n'.format(stdClippedRatio))
outLines.append('Mean RA offset (1 - 2): {0} degrees\n'.format(meanRAOffset))
outLines.append('Std. dev. RA offset (1 - 2): {0} degrees\n'.format(stdRAOffset))
outLines.append('Mean 3-sigma-clipped RA offset (1 - 2): {0} degrees\n'.format(meanClippedRAOffset))
outLines.append('Std. dev. 3-sigma-clipped RA offset (1 - 2): {0} degrees\n\n'.format(stdClippedRAOffset))
outLines.append('Mean Dec offset (1 - 2): {0} degrees\n'.format(meanDecOffset))
outLines.append('Std. dev. Dec offset (1 - 2): {0} degrees\n'.format(stdDecOffset))
outLines.append('Mean 3-sigma-clipped Dec offset (1 - 2): {0} degrees\n'.format(meanClippedDecOffset))
outLines.append('Std. dev. 3-sigma-clipped Dec offset (1 - 2): {0} degrees\n\n'.format(stdClippedDecOffset))
fileName = outDir+'stats.txt'
statsFile = open(fileName, 'w')
statsFile.writelines(outLines)
statsFile.close()
return stats
def instantiate(self, *args):
'''
Clear the old NuisanceSampler, instantiate the regions using the stored
residual spectra, and create a new NuisanceSampler.
'''
# threshold for sigma clipping
sigma = config["sigma_clip"]
# array that specifies if a pixel is already covered.
# to start, it should be all False
covered = np.zeros((self.npoints, ), dtype='bool')
#average all of the spectra in the deque together
residual_array = np.array(self.resid_deque)
if len(self.resid_deque) == 0:
raise RuntimeError("No residual spectra stored yet.")
else:
residuals = np.average(residual_array, axis=0)
# run the sigma_clip algorithm until converged, and we've identified the outliers
filtered_data = sigma_clip(residuals, sig=sigma, iters=None)
mask = filtered_data.mask
wl = self.wl
sigma0 = config['region_priors']['sigma0']
logAmp = config["region_params"]["logAmp"]
sigma = config["region_params"]["sigma"]
# Sort in decreasing strength of residual
self.nregions = 0
regions = {}
region_mus = {}
for w, resid in sorted(zip(wl[mask], np.abs(residuals[mask])),
key=itemgetter(1),
reverse=True):
if w in wl[covered]:
continue
else:
# check to make sure region is not *right* at the edge of the echelle order
if w <= np.min(wl) or w >= np.max(wl):
continue
else:
# instantiate region and update coverage
# Default amp and sigma values
regions[self.nregions] = {
"logAmp": logAmp,
"sigma": sigma,
"mu": w
}
region_mus[
self.nregions] = w # for evaluating the mu prior
self.nregions += 1
# determine the stretch of wl covered by this new region
ind = (wl >= (w - sigma0)) & (wl <= (w + sigma0))
# update the covered regions
covered = covered | ind
# Take the current nuisance positions as a starting point, and add the regions
starting_dict = self.sampler.params.copy()
starting_dict["regions"] = regions
region_mus = np.array([region_mus[i] for i in range(self.nregions)])
# Setup the priors
region_priors = config["region_priors"]
region_priors.update({"mus": region_mus})
prior_params = {"regions": region_priors}
# do all this crap again
cheb_MH_cov = float(config["cheb_jump"])**2 * np.ones((self.npoly, ))
cov_MH_cov = np.array([
float(config["cov_jump"][key]) for key in self.sampler.cov_tup
])**2
region_MH_cov = [
float(config["region_jump"][key])**2
for key in C.cov_region_parameters
]
regions_MH_cov = np.array(
[region_MH_cov for i in range(self.nregions)]).flatten()
nuisance_MH_cov = np.diag(
np.concatenate((cheb_MH_cov, cov_MH_cov, regions_MH_cov)))
print(starting_dict)
print("cov shape {}".format(nuisance_MH_cov.shape))
# Initialize a new sampler, replacing the old one
self.sampler = NuisanceSampler(OrderModel=self,
starting_param_dict=starting_dict,
cov=nuisance_MH_cov,
debug=True,
outdir=self.noutdir,
prior_params=prior_params,
order=self.order)
self.p0 = self.sampler.p0
# Update the nuisance parameters to the starting values so that we at least have a self.data_mat
print("Updating nuisance parameter data products to starting values.")
self.update_nuisance(starting_dict)
self.lnprob = self.evaluate()
# To speed up convergence, try just doing a bunch of nuisance runs before
# going into the iteration pattern
print("Doing nuisance burn-in for {} samples".format(
config["nuisance_burn"]))
self.independent_sample(config["nuisance_burn"])
def ubersmoother(self,check=False):
def plot_ubersmoother(x, y, idx, idx0, smooth=False, labels=None):
# Plot various steps of the smoothing process
ax.scatter(x, y, s=60, alpha=0.2)
try:
idx0_len = len(idx0)
except:
idx0_len = 0
if idx0_len > 0:
ax.scatter(x[idx0], y[idx0],
marker='x', c='black', lw=1.5, s=60,
label='removed previously')
ax.scatter(x[idx], y[idx],
marker='x', c='red', lw=2.5, s=60,
label='removed this loop')
if smooth:
ax.plot(x,np.interp(x, x[~idx], y[~idx]),
c='orange', lw=2,
label='smoothed')
# Set axis
ax.axis([np.min(x), np.max(x), np.min(y)*0.98, np.max(y)*1.02])
# Set labels
ax.legend(loc='best')
ax.set_title(labels[0])
ax.set_xlabel(labels[1])
ax.set_ylabel(labels[2])
#######################
# Initializing variables
time = self.time
phase = self.phase
flux = self.flux
ferr = self.ferr
# Find the number of full periods in lightcurve
n_epochs = float(len(flux))
n_epochs_period = float(len(np.where((time >= time[0]) &
(time < time[0]+period))[0]))
n_full_periods = np.ceil(n_epochs / n_epochs_period)
# Append nans to flux so there an integer number of periods
n_pad = (n_epochs_period*n_full_periods) - n_epochs
ind_pad = np.append(range(0,len(flux)),[-1]*n_pad)
ind_reshape = np.reshape(ind_pad,(n_full_periods,n_epochs_period))
flux_smooth = np.array(flux[:])
outlier_mask = np.zeros(flux.shape,dtype=bool)
###########################
# SMOOTHING THE LIGHTCURVE:
# 1) Slice by period-width slices (starting at 1 period)
# 2) Sort the flux by phase
# A. if slice width == period: rolling median
# B. if slice width > period: median_filtering + sigma_clipping
# 3) Shift slice over one period, repeat steps.
# 4) Store outliers in mask over which to interpolate later.
# width is 1 period, median filtering+sigma_clipping if > 1 period)
# Loop up to slices of 10*period OR
# if fewer than 10 periods, # of periods
if n_full_periods < 10:
n_slices = n_full_periods
else: n_slices = 10
if check:
plt_ct = 1
for p in np.arange(1,n_slices):
for s in np.arange(0,n_full_periods-p):
indices = [0,p]+s
# Start and end indices
start_INDX = indices[0].astype("int")
end_INDX = indices[1].astype("int")
# Select region of interest
ind_region = ind_reshape[start_INDX:end_INDX].flatten()
msk = ind_region < 0
ind_region = ind_region[~msk]
flux_region = flux_smooth[ind_region]
time_region = time[ind_region]
phase_region = phase[ind_region]
# Phase the data
phasesort = np.argsort(phase_region)
flux_region_phasesort = flux_region[phasesort]
time_region_phasesort = time_region[phasesort]
phase_region_phasesort = phase_region[phasesort]
ind_region_phasesort = ind_region[phasesort]
if p == 1:
#########################
### Rolling median method
threshold = 3.
flux_pandas = rolling_median(flux_region_phasesort,
window=5, center=True, freq=period)
diff = np.abs(flux_region_phasesort - flux_pandas)
outlier_idx = diff > np.std(diff[~np.isnan(diff)])
outlier_mask[ind_region_phasesort] = outlier_idx
if check and s==0:
fig = plt.figure(figsize=(12,8))
ax = fig.add_subplot(111)
labels = ['Rolling Median on 1-folded Period Slices',
'Phase',
'Flux (counts)']
plot_ubersmoother(phase_region_phasesort,
flux_region_phasesort,
outlier_idx,
None,
labels=labels)
plt.show()
else:
###################################
# Median_filtering + sigma_clipping.
# Median_filtering boxsize = 3% of period (scales with npts)
box = round(len(phase_region_phasesort)*0.03)
flux_region_filt = median_filter(flux_region_phasesort,
size=box, mode="reflect")#,
#cval=np.median(flux_region_phasesort))
clip = sigma_clip(flux_region_phasesort - flux_region_filt, 2.0)
# Sort clip into time spacing and save outliers in outliermask
timesort = np.argsort(time_region_phasesort)
clip_timesort = clip[timesort]
outlier_mask[ind_region[np.where(clip_timesort.mask == True)[0]]] = True
if check and s==0 and p in [2,9]:
if plt_ct == 1:
fig = plt.figure(figsize=(12,12))
ax = fig.add_subplot(2,1,plt_ct)
labels = ['Sigma Clipping on %s-folded Period Slices' %(p),
'Phase',
'Flux (counts)']
plot_ubersmoother(phase_region_phasesort,
flux_region_phasesort,
clip.mask,
outlier_mask[ind_region_phasesort],
labels=labels)
ax.plot(phase_region_phasesort, flux_region_filt,
c='red', lw=1.5)
if plt_ct == 3:
fig.subplots_adjust(hspace=0.40)
plt.show()
else:
plt_ct+=1
# Interpolate across missing points
flux_smooth = np.interp(time, time[~outlier_mask], flux[~outlier_mask])
# If long-cadence data, insert raw transits, apply gaussian filter to
# smooth. Gaussian_filter causes us to underestimate the bottoms of
# the wd occultations.
if "LC" in self.filename:
only_transit = np.where((phase >= 0.73) & (phase <= 0.77))[0]
flux_smooth[only_transit] = flux[only_transit]
flux_smooth_final = gaussian_filter(flux_smooth, 1, order=0)
# For the short-cadence data, median_filter (290) entire lightcurve,
# then insert median_filtered (23 window) wd occultations
elif "SC" in self.filename:
only_transit = np.where((phase >= 0.74) & (phase <= 0.78))[0]
flux_smooth_no_transit = median_filter(flux_smooth, size=290)
flux_smooth_transit = median_filter(flux_smooth, size=23)
flux_smooth_no_transit[only_transit] = flux_smooth_transit[only_transit]
flux_smooth_final = flux_smooth_no_transit
if check:
fig = plt.figure(figsize=(12,8))
ax = fig.add_subplot(111)
ax.plot(time, flux, '-ko', lw=2, alpha=0.6)
ax.plot(time, flux_smooth_final, c="#32cd32", lw=2, alpha=0.7)
ax.axis([np.min(time), np.max(time),
np.min(flux)*0.98, np.max(flux)*1.02])
ax.set_title('Final Smoothed Flux')
ax.set_xlabel('Time (days)')
ax.set_ylabel('Flux (counts')
plt.show()
self.flux_smooth = flux_smooth_final
self.outlier_mask = outlier_mask
# Use smoothed flux to normalize. Mask out occultations
self.normalize()
def correct_overscan(amplifier, bias_slice=None, data_slice=None, clip=3.):
"""Extract bias-corrected, exposed parts of raw GMOS fits file
Bias is determined from selected regions of overscan, clipping outliers
Parameters
----------
amplifier: ~fits.ImageHDU
bias_slice: list of 2 slices
slices in Y, X to use for determining the bias level
(default=None: use 'BIASSEC' from header)
data_slice: list of 2 slices
slices in Y, X that give good parts of the array
(default=None: use 'DATASEC' from header)
clip: ~float
number of standard deviations from the median within which overscan
values have to be to be included.
Returns
-------
~fits.ImageHDU
bias-corrected exposed parts
Note
----
The output fits structure will have the following new headers:
APPLIED DATASEC: section of initial data array extracted
APPLIED BIASSEC: section of initial data array used to determine bias level
BIAS: bias value subtracted (ADU)
BIASSTD: standard deviation of values going in to bias value (ADU)
(should be similar to read noise; this is not checked here)
BIASUNC: formal uncertainty in bias (BIASSTD/sqrt(N), in ADU)
The following headers are updated
CRPIX1: X reference pixel for the world coordinate system
CRPIX1: Y reference pixel for the world coordinate system
CCDSEC: the part of the chip represented by the data section
DETSEC: the part of the overall array represented by the data section
"""
if bias_slice is None:
bias_slice = sec2slice(amplifier.header['BIASSEC'])
if data_slice is None:
data_slice = sec2slice(amplifier.header['DATASEC'])
data_subslice = None
else:
# ensure we get numbers only, rather than relative indices,
# otherwise cannot store it as DATASEC header later
data_slice = [slice(*s.indices(shape))
for s, shape in zip(data_slice, amplifier.data.shape)]
# get equivalent subslice of 'DATASEC' for updating CCDSEC, DETSEC
data_subslice = get_subslices(sec2slice(amplifier.header['DATASEC']),
data_slice)
overscan = amplifier.data[bias_slice]
if clip:
clipped = stats.sigma_clip(overscan, clip, 1)
else:
clipped = overscan
bias_estimate, bias_std = clipped.mean(), clipped.std()
bias_unc = bias_std/np.sqrt(clipped.size)
outamp = fits.ImageHDU(amplifier.data[data_slice]-bias_estimate,
amplifier.header)
# add new headers with information on what was done
outamp.header['APPLIED DATASEC'] \
= slice2sec(data_slice), 'Section considered good'
outamp.header['APPLIED BIASSEC'] \
= slice2sec(bias_slice), 'Section used to estimate bias'
outamp.header['BIAS'] = bias_estimate, 'ADU'
outamp.header['BIASSTD'] = bias_std, 'ADU'
outamp.header['BIASUNC'] = bias_unc, 'ADU'
# adjust remove/existing headers as appropriate
outamp.header.remove('BIASSEC') # no BIAS section left
outamp.header.remove('DATASEC') # all of image is now DATA
outamp.header['CRPIX1'] += data_slice[1].start
outamp.header['CRPIX2'] += data_slice[0].start
if data_subslice:
binning = np.fromstring(amplifier.header['CCDSUM'], sep=' ',
dtype=np.int)
ccd_subslice = adjust_subslices(data_subslice, binning)
outamp.header['CCDSEC'] = slice2sec(slice_slices(
sec2slice(amplifier.header['CCDSEC']), ccd_subslice))
outamp.header['DETSEC'] = slice2sec(slice_slices(
sec2slice(amplifier.header['DETSEC']), ccd_subslice))
return outamp
beforeImg[~stableMask] = 0
afterImg[~stableMask] = 0
print 'masked stable sdev<{}%'.format(sdevThreshold)
bCoolMask = False
if bCoolMask:
effIntImg = np.load('stableMask.npz')['effIntImg']
effIntThreshold = .999
mask = effIntImg >= effIntThreshold
beforeImg[~mask] = 0
afterImg[~mask] = 0
print 'masked effIntTime fraction >{}'.format(effIntThreshold)
nRows,nCols = np.shape(beforeImg)
clippedBeforeImg = sigma_clip(beforeImg,sig=3)
clipMask = clippedBeforeImg.mask
beforeImg[clipMask] = 0
afterImg[clipMask] = 0
deadBeforeImg = (beforeImg == 0)
deadAfterImg = (afterImg == 0)
beforeList = beforeImg[beforeImg != 0]
afterList = afterImg[afterImg != 0]
afterImg[deadAfterImg] = np.nan
beforeImg[deadBeforeImg] = np.nan
plotArray(title='without flatcal',image=beforeImg)
#plotArray(title='with flatcal',image=afterImg)
def plotFluxRatiosDist(predFlux,
measFlux,
RA,
Dec,
refRA,
refDec,
labels,
outDir,
name1,
name2,
format,
clip=True):
"""
Makes plot of measured-to-predicted flux ratio vs. distance from center
"""
import numpy as np
from ..operations_lib import calculateSeparation
try:
from astropy.stats.funcs import sigma_clip
except ImportError:
from astropy.stats import sigma_clip
try:
import matplotlib
if matplotlib.get_backend() is not 'Agg':
matplotlib.use("Agg")
import matplotlib.pyplot as plt
from matplotlib.ticker import FuncFormatter
except Exception as e:
raise ImportError('PyPlot could not be imported. Plotting is not '
'available: {0}'.format(e.message))
if name1 is None:
name1 = 'Model 1'
if name2 is None:
name2 = 'Model 2'
ratio = measFlux / predFlux
separation = np.zeros(len(measFlux))
for i in range(len(measFlux)):
separation[i] = calculateSeparation(RA[i], Dec[i], refRA, refDec).value
fig = plt.figure(figsize=(7.0, 5.0))
ax1 = plt.subplot(1, 1, 1)
ax1.plot(separation, ratio, 'o')
plt.title('Flux Density Ratios ({0} / {1})'.format(name1, name2))
plt.ylabel('Flux density ratio')
plt.xlabel('Distance from center (deg)')
# Calculate mean ratio and std dev.
if clip:
mean_ratio = np.mean(sigma_clip(ratio))
std = np.std(sigma_clip(ratio))
else:
mean_ratio = np.mean(ratio)
std = np.std(ratio)
ax1.set_ylim(0, 2.0 * mean_ratio)
xmin, xmax, ymin, ymax = plt.axis()
ax1.plot([0.0, xmax], [mean_ratio, mean_ratio], '--g')
ax1.plot([0.0, xmax], [mean_ratio + std, mean_ratio + std], '-.g')
ax1.plot([0.0, xmax], [mean_ratio - std, mean_ratio - std], '-.g')
if labels is not None:
xls = separation
yls = ratio
for label, xl, yl in zip(labels, xls, yls):
plt.annotate(label,
xy=(xl, yl),
xytext=(-2, 2),
textcoords='offset points',
ha='right',
va='bottom')
plt.savefig(outDir + 'flux_ratio_vs_distance.{}'.format(format),
format=format)
def collapse(array_in, region=None, method='sum', sigma=False):
"""
Collapse a slice of a datacube, with a given mode, along the
wavelength slice.
This function handles both numpy.array (with no sigma clipping)
and numpy.ma masked arrays (with sigma clipping, from astropy.stast.funcs)
Parameters
----------
array_in : numpy.ndarray
The input datacube array.
region : The region of the datacube to collapse, in the form of [a, b],
where a is the 1st slice boundary, and b is the 2nd + 1 slice boundary.
Can be omitted to collapse entire z-axis.
method : {'median', 'sum', 'mean'}
The method of collapse.
sigma : bool, optional
Flag for wether to perform sigma clipping or not (default is False).
Returns
-------
collapsedArray : numpy.ndarray
Collapsed image.
Example usage:
1. >> result = ifu_3d_collapse(cube, region=[1000, 1999], method='median')
Median collapse cube from z-axis frames 1000-2000.
2. >> result = ifu_3d_collapsed(cube, method='mean', sigma=2.5)
Mean collapse a cube along the entire z-axis, performing a sigma
clipping of 2.5.
"""
# Extract the desired slice...
if not region:
print('(3d_collapse): Entire z-axis will be collapsed!')
slice_array = array_in
else:
print('(3d_collapse): z-axis collapse limits: ', region)
slice_array = array_in[region[0]:region[1], :, :]
# ... and perform the desired operation, based on input mode.
collapsed_array = _arrayCollapse(slice_array, method=method)
if sigma:
# Import sigma_clip from astropy
from astropy.stats.funcs import sigma_clip
# Perform sigma clipping on input cube
print('(3d_collapse): Clipping data ...')
clipped = sigma_clip(slice_array, sigma, iters=None, axis=0, copy=True)
# Compare the clipped sample to determine wether we need to
# recalculate the collapse
if np.array_equal(clipped.compressed(), slice_array.flatten()):
print('Arrays are exactly the same, no clipping required.')
else:
# Now that we have the sigma mask, recalculate the collapse...
collapsed_array = _maskedCollapse(clipped, method)
# Returns the collapsed array
#print('(3d_collapse): Shape of returned array:', collapsed_array.shape)
return collapsed_array
def findStats(predFlux, measFlux, RA, Dec, refRA, refDec, outDir, info0, info1,
info2, name1, name2):
"""
Calculates statistics and saves them to 'stats.txt'
"""
import os
import numpy as np
from ..operations_lib import calculateSeparation
try:
from astropy.stats.funcs import sigma_clip
except ImportError:
from astropy.stats import sigma_clip
ratio = measFlux / predFlux
meanRatio = np.mean(ratio)
stdRatio = np.std(ratio)
clippedRatio = sigma_clip(ratio)
meanClippedRatio = np.mean(clippedRatio)
stdClippedRatio = np.std(clippedRatio)
RAOffsets = np.zeros(len(RA))
DecOffsets = np.zeros(len(Dec))
for i in range(len(RA)):
if RA[i] >= refRA[i]:
sign = 1.0
else:
sign = -1.0
RAOffsets[i] = sign * calculateSeparation(RA[i], Dec[i], refRA[i],
Dec[i]).value # deg
if Dec[i] >= refDec[i]:
sign = 1.0
else:
sign = -1.0
DecOffsets[i] = sign * calculateSeparation(RA[i], Dec[i], RA[i],
refDec[i]).value # deg
meanRAOffset = np.mean(RAOffsets)
stdRAOffset = np.std(RAOffsets)
clippedRAOffsets = sigma_clip(RAOffsets)
meanClippedRAOffset = np.mean(clippedRAOffsets)
stdClippedRAOffset = np.std(clippedRAOffsets)
meanDecOffset = np.mean(DecOffsets)
stdDecOffset = np.std(DecOffsets)
clippedDecOffsets = sigma_clip(DecOffsets)
meanClippedDecOffset = np.mean(clippedDecOffsets)
stdClippedDecOffset = np.std(clippedDecOffsets)
stats = {
'meanRatio': meanRatio,
'stdRatio': stdRatio,
'meanClippedRatio': meanClippedRatio,
'stdClippedRatio': stdClippedRatio,
'meanRAOffsetDeg': meanRAOffset,
'stdRAOffsetDeg': stdRAOffset,
'meanClippedRAOffsetDeg': meanClippedRAOffset,
'stdClippedRAOffsetDeg': stdClippedRAOffset,
'meanDecOffsetDeg': meanDecOffset,
'stdDecOffsetDeg': stdDecOffset,
'meanClippedDecOffsetDeg': meanClippedDecOffset,
'stdClippedDecOffsetDeg': stdClippedDecOffset
}
outLines = ['Statistics from sky model comparison\n']
outLines.append('------------------------------------\n\n')
outLines.append('Sky model 1 ({}):\n'.format(name1))
outLines.append(info1 + '\n\n')
outLines.append('Sky model 2 ({}):\n'.format(name2))
outLines.append(info2 + '\n\n')
outLines.append(info0 + '\n')
outLines.append('Number of matches found for comparison: {0}\n\n'.format(
len(predFlux)))
outLines.append('Mean flux density ratio (1 / 2): {0}\n'.format(meanRatio))
outLines.append(
'Std. dev. flux density ratio (1 / 2): {0}\n'.format(stdRatio))
outLines.append(
'Mean 3-sigma-clipped flux density ratio (1 / 2): {0}\n'.format(
meanClippedRatio))
outLines.append(
'Std. dev. 3-sigma-clipped flux density ratio (1 / 2): {0}\n\n'.format(
stdClippedRatio))
outLines.append(
'Mean RA offset (1 - 2): {0} degrees\n'.format(meanRAOffset))
outLines.append(
'Std. dev. RA offset (1 - 2): {0} degrees\n'.format(stdRAOffset))
outLines.append(
'Mean 3-sigma-clipped RA offset (1 - 2): {0} degrees\n'.format(
meanClippedRAOffset))
outLines.append(
'Std. dev. 3-sigma-clipped RA offset (1 - 2): {0} degrees\n\n'.format(
stdClippedRAOffset))
outLines.append(
'Mean Dec offset (1 - 2): {0} degrees\n'.format(meanDecOffset))
outLines.append(
'Std. dev. Dec offset (1 - 2): {0} degrees\n'.format(stdDecOffset))
outLines.append(
'Mean 3-sigma-clipped Dec offset (1 - 2): {0} degrees\n'.format(
meanClippedDecOffset))
outLines.append(
'Std. dev. 3-sigma-clipped Dec offset (1 - 2): {0} degrees\n\n'.format(
stdClippedDecOffset))
fileName = outDir + 'stats.txt'
statsFile = open(fileName, 'w')
statsFile.writelines(outLines)
statsFile.close()
return stats
def instantiate(self, *args):
'''
Clear the old NuisanceSampler, instantiate the regions using the stored
residual spectra, and create a new NuisanceSampler.
'''
# threshold for sigma clipping
sigma=config["sigma_clip"]
# array that specifies if a pixel is already covered.
# to start, it should be all False
covered = np.zeros((self.npoints,), dtype='bool')
#average all of the spectra in the deque together
residual_array = np.array(self.resid_deque)
if len(self.resid_deque) == 0:
raise RuntimeError("No residual spectra stored yet.")
else:
residuals = np.average(residual_array, axis=0)
# run the sigma_clip algorithm until converged, and we've identified the outliers
filtered_data = sigma_clip(residuals, sig=sigma, iters=None)
mask = filtered_data.mask
wl = self.wl
sigma0 = config['region_priors']['sigma0']
logAmp = config["region_params"]["logAmp"]
sigma = config["region_params"]["sigma"]
# Sort in decreasing strength of residual
self.nregions = 0
regions = {}
region_mus = {}
for w, resid in sorted(zip(wl[mask], np.abs(residuals[mask])), key=itemgetter(1), reverse=True):
if w in wl[covered]:
continue
else:
# check to make sure region is not *right* at the edge of the echelle order
if w <= np.min(wl) or w >= np.max(wl):
continue
else:
# instantiate region and update coverage
# Default amp and sigma values
regions[self.nregions] = {"logAmp":logAmp, "sigma":sigma, "mu":w}
region_mus[self.nregions] = w # for evaluating the mu prior
self.nregions += 1
# determine the stretch of wl covered by this new region
ind = (wl >= (w - sigma0)) & (wl <= (w + sigma0))
# update the covered regions
covered = covered | ind
# Take the current nuisance positions as a starting point, and add the regions
starting_dict = self.sampler.params.copy()
starting_dict["regions"] = regions
region_mus = np.array([region_mus[i] for i in range(self.nregions)])
# Setup the priors
region_priors = config["region_priors"]
region_priors.update({"mus":region_mus})
prior_params = {"regions":region_priors}
# do all this crap again
cheb_MH_cov = float(config["cheb_jump"])**2 * np.ones((self.npoly,))
cov_MH_cov = np.array([float(config["cov_jump"][key]) for key in self.sampler.cov_tup])**2
region_MH_cov = [float(config["region_jump"][key])**2 for key in C.cov_region_parameters]
regions_MH_cov = np.array([region_MH_cov for i in range(self.nregions)]).flatten()
nuisance_MH_cov = np.diag(np.concatenate((cheb_MH_cov, cov_MH_cov, regions_MH_cov)))
print(starting_dict)
print("cov shape {}".format(nuisance_MH_cov.shape))
# Initialize a new sampler, replacing the old one
self.sampler = NuisanceSampler(OrderModel=self, starting_param_dict=starting_dict, cov=nuisance_MH_cov,
debug=True, outdir=self.noutdir, prior_params=prior_params, order=self.order)
self.p0 = self.sampler.p0
# Update the nuisance parameters to the starting values so that we at least have a self.data_mat
print("Updating nuisance parameter data products to starting values.")
self.update_nuisance(starting_dict)
self.lnprob = self.evaluate()
# To speed up convergence, try just doing a bunch of nuisance runs before
# going into the iteration pattern
print("Doing nuisance burn-in for {} samples".format(config["nuisance_burn"]))
self.independent_sample(config["nuisance_burn"])