def _sigclip_data(self, data_ma):
"""
Perform sigma clipping on the data in regions of size
``box_shape``.
"""
ny, nx = data_ma.shape
ny_box, nx_box = self.box_shape
y_nbins = int(ny / ny_box) # always integer because data were padded
x_nbins = int(nx / nx_box) # always integer because data were padded
data_rebin = np.ma.swapaxes(data_ma.reshape(
y_nbins, ny_box, x_nbins, nx_box), 1, 2).reshape(y_nbins, x_nbins,
ny_box * nx_box)
del data_ma
with warnings.catch_warnings():
warnings.simplefilter('ignore')
if ASTROPY_LT_1P1:
self.data_sigclip = sigma_clip(
data_rebin, sig=self.sigclip_sigma, axis=2,
iters=self.sigclip_iters, cenfunc=np.ma.median,
varfunc=np.ma.var)
else:
self.data_sigclip = sigma_clip(
data_rebin, sigma=self.sigclip_sigma, axis=2,
iters=self.sigclip_iters, cenfunc=np.ma.median,
stdfunc=np.std)
del data_rebin
def _calculate_collapse(self, data_name, operation, spatial_region, sigma_selection, sigma_parameter, start_index, end_index):
start_wavelength = self.wavelengths[start_index]
end_wavelength = self.wavelengths[end_index]
label = '{}-collapse-{} ({:.4e}, {:.4e})'.format(data_name, operation,
start_wavelength,
end_wavelength)
# Setup the input_data (and apply the spatial mask based on
# the selection in the spatial_region_combobox
input_data = self.data[data_name]
log.debug(' spatial region is {}'.format(spatial_region))
if not spatial_region == 'Image':
subset = [x.to_mask() for x in self.data.subsets if x.label == spatial_region][0]
input_data = input_data * subset
# Apply sigma clipping
if 'Simple' in sigma_selection:
sigma = sigma_parameter
if sigma is None:
return
input_data = sigma_clip(input_data, sigma=sigma, axis=0)
label += ' sigma={}'.format(sigma)
elif 'Advanced' in sigma_selection:
sigma, sigma_lower, sigma_upper, sigma_iters = sigma_parameter
log.debug(' returned from calculate_callback_advanced_sigma_check with sigma {} sigma_lower {} sigma_upper {} sigma_iters {}'.format(
sigma, sigma_lower, sigma_upper, sigma_iters))
if sigma is None:
return
input_data = sigma_clip(input_data, sigma=sigma, sigma_lower=sigma_lower,
sigma_upper=sigma_upper, iters=sigma_iters, axis=0)
# Add to label so it is clear which overlay/component is which
if sigma:
label += ' sigma={}'.format(sigma)
if sigma_lower:
label += ' sigma_lower={}'.format(sigma_lower)
if sigma_upper:
label += ' sigma_upper={}'.format(sigma_upper)
if sigma_iters:
label += ' sigma_iters={}'.format(sigma_iters)
else:
input_data = input_data # noop
# Do calculation if we got this far
new_wavelengths, new_component = collapse_cube(input_data, data_name, self.data.coords.wcs,
operation, start_index, end_index)
new_component_unit = self.data.get_component(data_name).units
return new_wavelengths, new_component, new_component_unit, label
def calc_stat(data, sigma=1.8, niter=10, algorithm='median'):
"""Calculate statistics for given data.
Parameters
----------
data : ndarray
Data to be calculated from.
sigma : float
Sigma for sigma clipping.
niter : int
Number of iterations for sigma clipping.
algorithm : {'mean', 'median', 'mode', 'stddev'}
Algorithm for statistics calculation.
Returns
-------
val : float
Statistics value.
Raises
------
ValueError
Invalid algorithm.
"""
arr = np.ravel(data)
if len(arr) < 1:
return 0.0
if ((astropy_version.major==1 and astropy_version.minor==0) or
(astropy_version.major < 1)):
arr_masked = sigma_clip(arr, sig=sigma, iters=niter)
else:
arr_masked = sigma_clip(arr, sigma=sigma, iters=niter)
arr = arr_masked.data[~arr_masked.mask]
if len(arr) < 1:
return 0.0
algorithm = algorithm.lower()
if algorithm == 'mean':
val = arr.mean()
elif algorithm == 'median':
val = np.median(arr)
elif algorithm == 'mode':
val = biweight_location(arr)
elif algorithm == 'stddev':
val = arr.std()
else:
raise ValueError('{0} is not a valid algorithm for sky background '
'calculations'.format(algorithm))
return val
def plotit(x,y,which,nbins=15,**kwargs):
x = x.clip(0,10)
y = y.clip(0,10)
x = sigma_clip(x,iters=2,sigma=5)
y = sigma_clip(y,iters=2,sigma=5)
ii = np.where(~x.mask & ~y.mask)
x = x[ii].filled()
y = y[ii].filled()
# xr,yr = {'optwise':[(0.0,1.3),(-0.02,0.08)]}[which]
n,xx,yy = np.histogram2d(x,y,nbins)#,[xr,yr])
plt.contour(n.T,extent=[xx[0],xx[-1],yy[0],yy[-1]],**kwargs)
def _sigclip_data(self):
"""
Perform sigma clipping on the (masked) data in regions of size
``box_size``.
"""
data3d = np.ma.swapaxes(self.data_ma.reshape(
self.nyboxes, self.box_size[0], self.nxboxes, self.box_size[1]),
1, 2).reshape(self.nyboxes, self.nxboxes, self.box_npts)
del self.data_ma
# the number of masked pixels in each mesh including *only* the
# input (and padding) mask
self._nmasked_mesh_orig = np.ma.count_masked(data3d, axis=2)
self.mesh_yidx, self.mesh_xidx = self._define_mesh2d_indices(
self._nmasked_mesh_orig)
data2d = data3d[self.mesh_yidx, self.mesh_xidx, :]
with warnings.catch_warnings():
warnings.simplefilter('ignore')
if ASTROPY_LT_1P1:
self.data_sigclip = sigma_clip(
data2d, sig=self.sigclip_sigma, axis=1,
iters=self.sigclip_iters, cenfunc=np.ma.median,
varfunc=np.ma.var)
else:
self.data_sigclip = sigma_clip(
data2d, sigma=self.sigclip_sigma, axis=1,
iters=self.sigclip_iters, cenfunc=np.ma.median,
stdfunc=np.std)
# the number of masked and unmasked pixels in each mesh
# including *both* the input (and padding) mask and pixels masked
# via sigma clipping
nmasked = np.ma.count_masked(self.data_sigclip, axis=1)
if self.remove_masked == 'threshold':
idx1d = np.where((self.box_npts - nmasked) >=
self.meshpix_threshold)
self.data_sigclip = self.data_sigclip[idx1d]
nmasked = nmasked[idx1d]
self.mesh_yidx = self.mesh_yidx[idx1d]
self.mesh_xidx = self.mesh_xidx[idx1d]
if len(self.mesh_yidx) == 0:
raise ValueError('There are no valid meshes available.')
self.nmasked_mesh = self._convert_1d_to_2d_mesh(nmasked)
self.nunmasked_mesh = self.box_npts - self.nmasked_mesh
return
def sigma_clip(self, data):
if not self.sigclip:
return data
if ASTROPY_LT_1P1:
warnings.warn('sigma_lower and sigma_upper will be ignored '
'because they are not supported astropy < 1.1',
AstropyUserWarning)
return sigma_clip(data, sig=self.sigma,
iters=self.iters)
else:
return sigma_clip(data, sigma=self.sigma,
sigma_lower=self.sigma_lower,
sigma_upper=self.sigma_upper,
iters=self.iters)
def plot_image(ax,data,size,alpha=0.5,cmap='cubehelix',extent=None): sigclip = sigma_clip(data,6) sigclip = sigclip.filled(np.nanmax(sigclip)) # fill clipped values with max ''' ### shift all numbers so that minimum = 0 try: minimum = sigclip[size[0]:size[1],size[2]:size[3]].min() except ValueError: print 1/0 if sigclip.min() < 0: sigclip += np.abs(minimum) else: sigclip -= np.abs(minimum) ''' ''' ### put into units of standard deviations ### start linear scale at 3sigma std = np.std(sigclip[size[2]:size[3],size[0]:size[1]]) sigclip /= std sigclip = np.clip(sigclip,1.0,np.inf) minimum = np.nanmin(sigclip) sigclip = sigclip - minimum ''' ### show image ax.imshow(sigclip, origin='lower', cmap=cmap,alpha=alpha,extent=extent)
def remove_aperture_outliers(self, aperture_centers, aperture_means, aperture_stddevs):
"""
This function ...
:return:
"""
means_distribution = Distribution.from_values(aperture_means, bins=50)
stddevs_distribution = Distribution.from_values(aperture_stddevs, bins=50)
#means_distribution.plot("Aperture means before sigma-clipping")
#stddevs_distribution.plot("Aperture stddevs before sigma-clipping")
clip_mask = stats.sigma_clip(aperture_stddevs, sigma=3.0, iters=None, copy=False).mask
clipped_aperture_centers = []
for i in range(len(clip_mask)):
if clip_mask[i]: continue
else: clipped_aperture_centers.append(aperture_centers[i])
aperture_centers = clipped_aperture_centers
aperture_means = np.ma.MaskedArray(aperture_means, clip_mask).compressed()
aperture_stddevs = np.ma.MaskedArray(aperture_stddevs, clip_mask).compressed()
means_distribution = Distribution.from_values(aperture_means, bins=50)
stddevs_distribution = Distribution.from_values(aperture_stddevs, bins=50)
#means_distribution.plot("Aperture means after sigma-clipping")
#stddevs_distribution.plot("Aperture stddevs after sigma-clipping")
# Return the sigma-clipped aperture properties
return aperture_centers, aperture_means, aperture_stddevs
def sigma_clip_mask(data, sigma_level=3.0, mask=None):
"""
This function ...
:param data:
:param sigma_level:
:param mask:
:return:
"""
# Split the x, y and z values of the data, without the masked values
x_values, y_values, z_values = general.split_xyz(data, mask=mask)
# Sigma-clip z-values that are outliers
masked_z_values = sigma_clip(z_values, sigma=sigma_level, iters=None, copy=False)
# Copy the mask or create a new one if none was provided
new_mask = copy.deepcopy(mask) if mask is not None else Mask(np.zeros_like(data))
for i, masked in enumerate(masked_z_values.mask):
if masked:
x = x_values[i]
y = y_values[i]
new_mask[y,x] = True
#if not isinstance(new_mask, Mask): print(new_mask, mask)
# Assert the mask is of type 'Mask'
assert isinstance(new_mask, Mask)
# Return the new or updated mask
return new_mask
def sigma_clip(values, sigma_level=3.0, return_nmasked=False, logarithmic=False):
"""
This function ...
:param values:
:param sigma_level:
:param return_nmasked:
:param logarithmic: the values are logarithmically distributed
:return:
"""
from astropy.stats import sigma_clip
# Sigma-clip
if logarithmic: values = np.log10(logarithmic)
masked_array = sigma_clip(values, sigma=sigma_level, iters=None)
# Get the number of masked values
nmasked = np.sum(masked_array.mask)
# Get the clipped list of values
clipped = list(masked_array.compressed())
# Return as list
if return_nmasked: return clipped, nmasked
else: return clipped
def cutoff(values, method, limit):
"""
This function ...
:param values:
:param method:
:param limit:
"""
# Percentage method
if method == "percentage":
# Create a sorted list for the input values
sorted_values = sorted(values)
# Determine the splitting point
split = (1.0-limit) * len(sorted_values)
index = int(round(split))
# Return the corresponding value in the sorted list
return sorted_values[index]
# Sigma-clipping method
elif method == "sigma_clip":
# Perform sigma clipping on the input list
masked_values = sigma_clip(np.array(values), sigma=limit, iters=None, copy=False)
# Calculate the maximum of the masked array
return np.ma.max(masked_values)
else: raise ValueError("Invalid cutoff method (must be 'percentage' or 'sigma_clip'")
def linear_bias_kappa(kt, kp):
kt = kt.ravel()
kp = kp.ravel()
kt = sigma_clip(kt, 8, 10)
kp = kp[~kt.mask]
kt = kt.compressed()
print kp.shape, kt.shape
bin = np.linspace(kp.min(), kp.max(), 30)
kp_b, kp_be, kt_b, kt_be, N, B = MyF.AvgQ(kp, kt, bin)
N[N == 0] = 1
kt_be /= np.sqrt(N)
b_init = [1]
kt_be[kt_be == 0] = 99.0
chi2 = lambda b: np.sum(((kt_b - kp_b * b[0]) / kt_be)**2)
bias = optimize.fmin(chi2, b_init)
print 'Bias ', 1/bias
chi2 = lambda b: np.sum(((kt_b - kp_b * b) / kt_be)**2)
m = minuit.Minuit(chi2)
m.migrad()
m.hesse()
b = m.values['b']
be = m.errors['b']
bias = 1/b
bias_e = be/b**2.
print 'b = %2.2f \pm %2.2f'%(bias, bias_e)
return bias, bias_e
def clipped(self, compress=False, inplace=False):
clip = sigma_clip(self, self.sigma)
if compress:
clip = clip.compress()
if not inplace:
return clip
self = clip
def plot_normalized_lightcurve(picid, lc0, psc_dir):
plt.figure(figsize=(12, 6))
i = 0
for color in 'rgb':
(lc0.loc[lc0.color == color].target + i).plot(marker='o', color=color, alpha=0.5)
(lc0.loc[lc0.color == color].reference + i).plot(marker='x', color=color, alpha=0.5)
i += .3
plt.title(f'Normalized Flux per channel (+ offset) - {picid}')
plt.legend()
plot_fn = os.path.join(psc_dir, f'normalized-flux-{picid}.png')
plt.savefig(plot_fn)
# Different
plt.figure(figsize=(12, 6))
i = 0
for color in 'rgb':
t0 = lc0.loc[lc0.color == color].target
r0 = lc0.loc[lc0.color == color].reference
f0 = sigma_clip(t0 / r0, sigma=3)
plt.plot((f0 + i), marker='o', ls='', alpha=0.5, color=color)
i += .1
# plt.ylim([.9, 1.1])
plt.title(f'Normalized Flux per channel (+ offset) - {picid}')
plt.legend()
plot_fn = os.path.join(psc_dir, f'normalized-flux-{picid}.png')
plt.savefig(plot_fn)
def masterfunc1(op): xax=np.arange(1.5,4.75,step=0.25) global allstackd1 global allstackd2 allstackd1=np.asarray(allstackd1) allstackd2=np.asarray(allstackd2) panel1,smooth1=highpassflist(allstackd1,op) panel2,smooth2=highpassflist(allstackd2,op) beforenan= np.argwhere(np.isnan(smooth2)) beforenan = collections.Counter([x for (x,y) in beforenan]) panel4= map(lambda t: sigma_clip(smooth2[t],sigma=5),np.arange(0,len(xax))) noisef1=np.ma.asarray(panel4) #panel3=np.ma.asarray(panel3) afternan = map( np.ma.count_masked,panel4) beforenan= map(lambda t: beforenan[t], np.arange(0,len(beforenan))) toss=np.asarray(afternan)-np.asarray(beforenan) panel4=map(normstar,panel4) panel4= map( lambda t: t-1, panel4) panel4=map(lambda t: np.sqrt(np.nanmean(np.square(t))), panel4) panel3=toss/(ct*64.0) panels=[] panels.append(panel1) panels.append(panel2) panels.append(panel3) panels.append(panel4) return panels
def filtered_selection(self,obsDb,ii):
# filter out bright sky values
t = join(obsDb[ii],self.sky,'frameIndex')
if len(t) < len(ii):
raise ValueError('missing files!')
#
badfield = np.in1d(obsDb['objName'][ii],
['rm10','rm11','rm12','rm13'])
badfield |= obsDb['fileName'][ii] == 'bokrm.20150405.0059'
fvar = t['skyRms']**2/t['skyMean']
badrms = sigma_clip(fvar,sigma=3.0,iters=2).mask
keep = (t['skyMean'] < self.maxCounts) & ~badrms & ~badfield
if keep.sum() < self.minNImg:
return ~badrms & ~badfield
#
grpNum = 0
pgrp = [grpNum]
for _i in range(1,len(ii)):
if obsDb['objName'][ii[_i]] != obsDb['objName'][ii[_i-1]]:
grpNum += 1
pgrp.append(grpNum)
pgrp = np.array(pgrp)
#
jj = np.where(keep)[0]
keep[jj] = False
jj2 = t['skyMean'][jj].argsort()[:self.maxNImg*2]
_,jj3 = np.unique(pgrp[jj[jj2]],return_index=True)
jj4 = t['skyMean'][jj[jj2[jj3]]].argsort()[:self.maxNImg]
keep[jj[jj2[jj3[jj4]]]] = True
return keep
def bin_every_n(x, start_idx, n = 10, reduction_func=lambda x: np.mean(x, 0)):
out = []
if x.ndim == 1:
x = x[:, None]
for i in range(start_idx, x.shape[0], n):
end_idx = min(i+n, x.shape[0])
out.append(st.sigma_clip(x[i:end_idx, :], sigma=2.5, iters=1, axis=0).mean(0))
return np.array(out)
def masterfunc1(op): xax=[2.5] global allstackd1 allstackd1=np.asarray(allstackd1) print 'allstackd1',len(allstackd1) panel1,panel2,smooth1=highpassflist(allstackd1,op) ''' panel1=map(normstar,flist1) panel1= map( lambda t: t-1, panel1) panel1=map(lambda t: np.sqrt(np.nanmean(np.square(t))), panel1) ''' smooth1=np.asarray(smooth1) beforenan= np.argwhere(np.isnan(smooth1)) beforenan=beforenan.flatten() panel3= sigma_clip(smooth1,sigma=5,cenfunc=np.nanmedian) p=np.ma.fix_invalid(panel3.data,panel3.mask,fill_value=0) tosstemp= [i for i, x in enumerate(panel3.mask) if x] panel3=np.asarray(panel3) for x in tosstemp: panel3[x]=np.nan afternan = np.ma.asarray(panel3) afternan = np.argwhere(np.isnan(afternan)) afternan = afternan.flatten() ''' smooth1=np.asarray(smooth1) beforenan= np.argwhere(np.isnan(smooth1)) beforenan=beforenan.flatten() panel3= sigma_clip(smooth1,sigma=5) afternan = np.asarray(panel3) afternan = np.argwhere(np.isnan(afternan)) afternan = afternan.flatten() ''' print beforenan print tosstemp print 'len of bef',len(beforenan) print 'len of toss temp',len(tosstemp) toss = set(tosstemp) - set(beforenan) #beforenan= map(lambda t: beforenan[t], np.arange(0,len(beforenan))) #toss=np.asarray(afternan)-np.asarray(beforenan) #print toss ''' panel3=map(normstar,panel3) panel3= map( lambda t: t-1, panel3) panel3=map(lambda t: np.sqrt(np.nanmean(np.square(t))), panel3) ''' panel2=np.asarray(panel2) panels=[] panels.append(panel1) panels.append(panel2) panels.append(smooth1) panels.append(panel3) return panels,toss
def plot_meds():
filename_cat= 'DES0555-5957_i_cat.fits'
filename_meds = 'DES0555-5957-i-meds-011.fits.fz'
medsobj = meds.MEDS(filename_meds)
sexcat = pyfits.getdata(filename_cat)
medcat = medsobj._cat
print 'n_objects meds ' , len(medcat)
print 'n_objects sexcat ' , len(sexcat)
select = sexcat['FLUXERR_AUTO'] > 1e-3
snr = sexcat['FLUX_AUTO'][select]/sexcat['FLUXERR_AUTO'][select]
pl.figure()
pl.hist(snr,bins=np.linspace(0,100))
select = (snr<50) * (snr>49)
high_snrs = np.nonzero(select)[0]
print 'len high_snrs' , len(high_snrs)
obj_id = high_snrs[2]
# obj_id = 6
print 'ncutout',medsobj._cat['ncutout'][obj_id]
print 'obj_id' , obj_id
mosaic = medsobj.get_mosaic(obj_id)
pl.figure()
pl.imshow(mosaic,interpolation='nearest')
pl.figure()
pl.plot(mosaic.flatten())
print 'np.std(mosaic.flatten())' , np.std(mosaic.flatten(),ddof=1)
print 'median_absolute_deviation' , median_absolute_deviation(mosaic.flatten())
clipped_mosaic,_ = sigma_clip(mosaic.flatten(),sig=2)
print 'np.std(clipped_mosaic)' , np.std(clipped_mosaic,ddof=1)
pl.figure()
pl.plot(clipped_mosaic.flatten())
pl.figure()
pl.hist(mosaic.flatten(),bins=100)
pl.show()
import pdb; pdb.set_trace()
def array_process(idata, sigcut, compress = False):
'''Create the sigma-clipped data array.'''
data = copy(idata)
scut = np.clip(sigcut, 1.0, sigcut) #if we clip too low, this will never end
good_data = sigma_clip(data, scut, iters=None)
if compress:
return good_data.compressed()
return good_data
def group_clip(data, v0):
""" Remove galaxies that are not members. """
idx = np.where(np.abs(data[3] - v0) < 1500)[0]
d = tuple([x[idx] for x in data])
newdata = sigma_clip(d[3], sigma=3, cenfunc=np.median,
stdfunc=funcs.biweight_midvariance,
copy=True, iters=10)
mask = ~newdata._mask
return tuple([x[mask] for x in d])
def average_background(bkg_list, sigma, maxiters):
"""
Average multiple background exposures into a combined data model
Parameters:
-----------
bkg_list: filename list
List of background exposure file names
Returns:
--------
avg_bkg: data model
The averaged background exposure
"""
num_bkg = len(bkg_list)
avg_bkg = None
cdata = None
# Loop over the images to be used as background
for i, bkg_file in enumerate(bkg_list):
log.debug(' Accumulate bkg from {}'.format(bkg_file))
bkg_model = datamodels.ImageModel(bkg_file)
# Initialize the avg_bkg model, if necessary
if avg_bkg is None:
avg_bkg = datamodels.ImageModel(bkg_model.shape)
if cdata is None:
cdata = np.zeros(((num_bkg,) + bkg_model.shape))
cerr = cdata.copy()
# Accumulate the data from this background image
cdata[i] = bkg_model.data
cerr[i] = bkg_model.err * bkg_model.err
avg_bkg.dq = np.bitwise_or(avg_bkg.dq, bkg_model.dq)
bkg_model.close()
# Clip the background data
log.debug(' clip with sigma={} maxiters={}'.format(sigma, maxiters))
mdata = sigma_clip(cdata, sigma=sigma, maxiters=maxiters, axis=0)
# Compute the mean of the non-clipped values
avg_bkg.data = mdata.mean(axis=0).data
# Mask the ERR values using the data mask
merr = np.ma.masked_array(cerr, mask=mdata.mask)
# Compute the combined ERR as the uncertainty in the mean
avg_bkg.err = np.sqrt(merr.sum(axis=0)) / (num_bkg - merr.mask.sum(axis=0))
return avg_bkg
def stack_spectra(df, colwave='wave', colf='fnu', colfu='fnu_u', colmask=[], output_wave_array=[], pre='f', sigmaclip=3) :
''' General-purpose function to stack spectra. Rebins wavelength.
Does not de-redshift spectra. If you want to stack in rest frame, run jrr.spec.convert2restframe_df(df) beforehand.
Any normalization by continuum should be done beforehand.
Input df{} is a dictionary of pandas data frames that contains the spectra.
colwave, colf, colfu, colmask, tell where to find the columns for wavelength, flux/flam/fnu, uncertainty, & input mask.
colmask is a column in dataframe of values to mask (True=masked)
Output wavelength array will be output_wave_array if it is supplied; else will use 1st spectrum in df{}.
'''
if len(output_wave_array) :
print("Caution: overriding the default wavelength range and dispersion!")
stacked = pandas.DataFrame(data=output_wave_array, columns=(colwave,))
else :
stacked = pandas.DataFrame(data=df[list(df.keys())[0]][colwave]) # Get output wavelength array from first spectrum
nbins = stacked.shape[0] #N of pixels
nspectra = len(df)
nf = np.ma.zeros(shape=(nspectra, nbins)) # temp array that will hold the input spectra
nf_u = np.ma.zeros(shape=(nspectra, nbins)) # using numpy masked arrays so can ignore nans from rebin_spec_new
for ii, spec in enumerate(iter(df.values())): # Rebin each spectrum (spec), and load all spectra into big fat arrays.
ma_spec = spec # masked version of spectrum
if colmask :
ma_spec.loc[ma_spec[colmask], colf] = np.nan # try setting to nan to solve the gap problem
#ma_spec.loc[ma_spec[colmask], colfu] = ma_spec[colf].median() * 1E6 # set this huge
nf[ii] = np.ma.masked_invalid(rebin_spec_new(ma_spec[colwave], ma_spec[colf], stacked[colwave], return_masked=True)) # fnu/flam rebinned
nf_u[ii] = np.ma.masked_invalid(rebin_spec_new(ma_spec[colwave], ma_spec[colfu], stacked[colwave], return_masked=True)) # uncertainty on above
nf[ :, 0:2].mask = True # mask the first 2 and last 2 pixels of each spectrum
nf[ :, -2:].mask = True
## NEED TO HAVE IT USE THE COLMASK, IF DEFINED.
# Is rebinning handling the uncertainties correctly?
stacked[pre+'sum'] = np.ma.sum(nf, axis=0)
stacked[pre+'sum_u'] = util.add_in_quad(nf_u, axis=0)
stacked[pre+'avg'] = np.ma.average(nf, axis=0)
stacked[pre+'avg_u'] = stacked[pre+'sum_u'] / np.ma.MaskedArray.count(nf, axis=0)
weights = nf_u ** -2 # compute the weighted avg
(stacked[pre+'weightavg'], sumweight) = np.ma.average(nf, axis=0, weights=weights, returned=True) # weighted avg
stacked[pre+'weightavg_u'] = sumweight**-0.5
nf_clip = sigma_clip(nf, sig=sigmaclip, iters=None, axis=0)
stacked[pre+'clipavg'], sumweight2 = np.ma.average(nf_clip, axis=0, weights=weights, returned=True)
stacked[pre+'clipavg_u'] = sumweight2**-0.5
stacked[pre+'median'] = np.ma.median(nf, axis=0)
stacked[pre+'medianxN'] = np.ma.median(nf, axis=0) * np.ma.MaskedArray.count(nf, axis=0)
stacked['Ngal'] = np.ma.MaskedArray.count(nf, axis=0) # How many spectra contribute to each wavelength
# compute the jackknife variance. Adapt from mage_stack_redo.py.
jackknife= np.ma.zeros(shape=(nspectra, nbins)) # The goal.
jack_var = np.ma.zeros(shape=nbins)
for ii in range(0, nspectra) :
jnf = nf.copy()
#print "DEBUGGING Jackknife, dropping ", ii, "from the stack"
jnf[ii, :].mask = True # Mask one spectrum
jackknife[ii], weight = np.ma.average(jnf, axis=0, weights=weights, returned=True) # all the work is done here.
jack_var = jack_var + (jackknife[ii] - stacked[pre+'weightavg'])**2
jack_var *= ((nspectra -1.0)/float(nspectra))
jack_std = np.sqrt(jack_var)
stacked[pre+'jack_std'] = jack_std
return(stacked, nf, nf_u)
def tossoutframe(allstackd): #5 sigmaclip of x0,y0,F allstackd=np.asarray(allstackd) temp=np.apply_along_axis(lambda m: convolve(m, Box1DKernel(50)), axis=0, arr=allstackd) #print temp[4,13:18,13:18] #Smoothening allstackd-=temp image_data1= sigma_clipping(temp) image_data2= bgsubtract(image_data1) cx, cy = centroid(image_data2) cx=sigma_clip(cx,sigma=5) cy=sigma_clip(cy,sigma=5) xax=np.arange(1.5,4.75,step=0.25) flist=map(lambda t: sigma_clip(aperphot(image_data2,t,cx,cy),sigma=5), xax) flist=np.asarray(flist) flist = map(normstar,flist) flist = map( lambda t: t-1, flist) flist=map(lambda t: np.sqrt(np.nanmean(np.square(t))), flist) return flist
def reject_outliers(rmcal,simdat,**kwargs):
sig = kwargs.get('reject_sig',3.0)
iters = kwargs.get('reject_niter',2)
for i,obj in enumerate(rmcal):
mags,errs = rmcal.get_object_phot(obj)
clipped = sigma_clip(mags,sig=sig,iters=iters)
# need a verbose argument
# if clipped.mask.sum() > mags.mask.sum():
# print 'object %d rejected %d' % (i,(clipped.mask&~mags.mask).sum())
obj.update_mask(clipped.mask)
def remove_outliers(self, sigma=5):
fmodel = self.flux_model(self.de.minimum_location)[0]
times, fluxes, pbids, errors = [], [], [], []
for i in range(len(self.times)):
res = self.fluxes[i] - fmodel[i]
mask = ~sigma_clip(res, sigma=sigma).mask
times.append(self.times[i][mask])
fluxes.append(self.fluxes[i][mask])
if self.errors is not None:
errors.append(self.errors[i][mask])
self._init_data(times, fluxes, self.pbids, (errors if self.errors is not None else None))
def __init__(self, filenames, hdu_num):
self.filenames = filenames
self.nfiles = len(filenames)
self.hdu_num = hdu_num
self.images = []
for filename in filenames:
img = RawImage(filename, hdu_num)
img.subtract_overscan()
self.images.append(img)
nx, ny = self.images[0].data.shape
cube = np.zeros((self.nfiles, nx, ny))
cube_weights = np.zeros_like(cube)
for j in range(self.nfiles):
cube[j, :, :] = self.images[j].data
cube_weights[j, :, :] = self.images[j].data_err ** -2.0
cubeclipped = stats.sigma_clip(cube, sig=3.0, cenfunc=np.mean, axis=0)
# Compute the weighted mean for each pixel in the stack, using
# only those weights for counts that weren't clipped.
cubemean = np.ma.average(cubeclipped, axis=0, weights=cube_weights)
# In the very unlikely (impossible?) event that all pixels in the
# stack were masked, assume zero counts.
cubemean = np.ma.filled(cubemean, 0.0)
self.data = cubemean
# Use pairwise differences of overscan-subtracted biases to estimate
# the error.
rmses = []
for i in range(0, self.nfiles - 1, 2):
diff = (
self.images[i + 1].data[self.images[i + 1].py_datasec] - self.images[i].data[self.images[i].py_datasec]
)
mn = np.mean(diff)
rms = np.sqrt(np.mean((diff - mn) ** 2.0))
rmses.append(rms)
mean_rms = np.mean(rmses)
self.data_err = np.full_like(self.data, mean_rms)
print "hdu %d, bias diff rms %f, scatter %f" % (
hdu_num,
mean_rms,
np.sqrt(np.mean((np.array(rmses) - mean_rms) ** 2.0)),
)
# Mask those pixels whose rms in the stack of unclipped pixels is
# greater than five times the rms measured from pairwise differences.
rms_arr = np.ma.average((cubeclipped - cubemean) ** 2.0, axis=0, weights=cube_weights)
rms_arr = np.sqrt(np.ma.filled(rms_arr, 0.0))
pixmask = np.zeros_like(cubemean, dtype=int)
bad_pix = ((rms_arr == 0.0) | (rms_arr > 5.0 * mean_rms)) & (pixmask == 0)
pixmask[bad_pix] = 1
self.data_mask = pixmask
def sigmaclip_limitsig(data,error=None,sig_limit=None,**kwd): # sigma clipping with limiting sigma data=np.array(data) mdata=ast.sigma_clip(data,**kwd) if sig_limit is not None: while(True): med=np.ma.median(mdata) sig=np.ma.std(mdata) if sig < sig_limit: break index=np.ma.argmax(np.ma.abs(mdata-med)) mdata.mask[index]=True return mdata
def lc_plot_diff_mag(self, result_file_path=None,
best_comparison_star=None,
mark_color="blue",
bar_color="red"):
print("Plotting asteroid's LC...")
fn = os.path.basename(result_file_path).split('.')[0]
# Two subplots, the axes array is 1-d
# Plotting settings
rcParams['figure.figsize'] = [10., 8.]
lc_ast_diff = plt.figure()
gs = gridspec.GridSpec(2, 1, height_ratios=[6, 2])
results = self.find_best_comp(result_file_path=result_file_path,
best_comparison_star=best_comparison_star)['with_mean_comp']
filtered_jd_vs_mag_diff = sigma_clip(results['t-c'],
sigma=3,
iters=10, stdfunc=mad_std)
# use only not rejected data (because umask used)
filtered_diff_umask = np.logical_not(filtered_jd_vs_mag_diff.mask)
# jd vs magt - magi
lc = lc_ast_diff.add_subplot(gs[0])
lc.set_title(fn)
lc.legend(loc=2, numpoints=1)
lc.grid(True)
lc.invert_yaxis()
lc.set_xlabel("$JD$", fontsize=12)
lc.set_ylabel("Diff Mag. ({0} - {1})".format(
results['ast_num'][0],
results['nomad1'][0]),
fontsize=12)
# Plotting settings
lc.errorbar(
results['jd'][filtered_diff_umask],
results['t-c'][filtered_diff_umask],
yerr=results['t-c-err'][filtered_diff_umask],
fmt='o',
ecolor=bar_color,
color=mark_color,
capsize=5,
elinewidth=2,
label='{0} - {1}'.format(fn, results['nomad1'][0]))
lc.legend(loc=2, numpoints=1)
lc_ast_diff.savefig("{0}/{1}_jd_vs_diff_mag_lc.pdf".format(os.getcwd(), fn))
def measure_image(image, wedge, clip_sigma=None):
"""Measure an image with the given wedge definition.
Parameters
----------
image : ndarray
The image to measure. It should be consistent with the image shape
expected by the :class:`WedgeBins` definition.
wedge : :class:`WedgeBins`
The binning defition.
clip_sigma : float
If set, this is the Gaussian standard deviation to apply sigma-clipping
against.
Returns
-------
profile : ndarray
A `numpy` structured array with the following fields:
- `area`, the area is square arcseconds
- `R`, the radial distance at the bin mid-point, in arcseconds
- `R_inner`, the radial distance at the inside edge, in arcseconds
- `R_outer`, the radial distance at the outside edge, in arcseconds
- `median`, the median pixel intensity in the bin
- `sigma`, the standard deviation of pixel intensities
"""
dt = [('area', np.float), ('R', np.float), ('R_inner', np.float),
('R_outer', np.float), ('median', np.float), ('sigma', np.float)]
profile = np.zeros(wedge.n_bins, dtype=np.dtype(dt))
for i, b in enumerate(wedge):
x1, x2 = b['xlim']
y1, y2 = b['ylim']
pixels = image[y1:y2, x1:x2].ravel()
good = np.where(np.isfinite(pixels))[0]
pixels = pixels[good]
if clip_sigma is not None:
# Perform sigma clipping
filtered = sigma_clip(pixels, sig=clip_sigma, iters=1)
good_only = filtered.data[~filtered.mask]
median = np.median(good_only)
sigma = np.std(good_only)
else:
# No sigma clipping
median = np.median(pixels)
sigma = np.std(pixels)
profile[i]['area'] = b['area']
profile[i]['R'] = b['r_mid']
profile[i]['R_inner'] = b['r_inner']
profile[i]['R_outer'] = b['r_outer']
profile[i]['median'] = median
profile[i]['sigma'] = sigma
return profile
fileids = np.array(range(len(ckpt)))[ckpt[:, 1] == '0']
nf = len(ckpt)
outfile = outdir + "/" + outname
for i, f in zip(fileids, files):
ticid = f.split("-")[2][-9:]
camp = f.split("-")[1][-2:]
print("computing {0} of {1} lightcurves\n TIC {2}".format(
i + 1, nf, ticid))
summaryfile = ticid + "_" + camp + "_summary.png"
cornerfile = ticid + "_" + camp + "_corner.png"
errfile = ticid + "_" + camp + "_err.dat"
lc = Lightcurve.tess(f)
try:
y = lc.flux - utils.trend(lc.t, lc.flux, 2)
mask = sigma_clip(y - medfilt(y, kernel_size=301), sigma=3)
x = lc.t[mask.mask == False]
y = y[mask.mask == False]
yerr = np.std(y - medfilt(y, kernel_size=51))
x = x[::5]
y = y[::5]
with pm.Model() as model:
mean = pm.Normal("mean", mu=np.mean(y), sd=np.std(y))
yerr = pm.Normal("yerr", mu=yerr, sd=5.0)
logamp = pm.Normal("logamp", mu=np.log(np.var(y)), sd=5)
period = utils.periodprior(lc.t, lc.flux, min_period=0.1)
logQ0 = pm.Uniform("logQ0", lower=-10, upper=10)
logdQ = pm.Normal("logdQ", mu=2.0, sd=5.0)
mix = pm.Uniform("mix", lower=0.0, upper=1.0)
def meas_sig_old(time, flux, weights, plot_q=False):
from astropy.stats import sigma_clip
df = np.diff(flux)
x = np.copy(time[:-1]) #np.arange(len(df))
dff1 = csaps.UnivariateCubicSmoothingSpline(
x,
df,
smooth=0.1,
weights=weights[:-1],
)(x)
# print('dff1', dff1)
dfn = df - dff1
dfnf = np.copy(dfn)
# plt.figure(figsize=(10, 4))
# plt.plot(x, df, '.k', label='Orig dF')
# plt.plot(x, dff1, '-c', label='dF fit 1')
# plt.figure(figsize=(10, 4))
# plt.plot(x, dfn, '.k', label='Orig dF')
for sd in [5]:
dfnf = sigma_clip(dfnf, sigma=sd, maxiters=2)
__, gi, __ = np.intersect1d(dfn, dfnf, return_indices=1)
gi = np.sort(gi)
# plt.figure(figsize=(10, 4))
# plt.plot(x[gi], dfn[gi], '.k', label='Orig dF')
# try:
dff = csaps.UnivariateCubicSmoothingSpline(
x[gi],
df[gi],
smooth=0.1,
weights=weights[:-1][gi],
)(x)
# except:
# print('FAIL', len(x), len(gi))
# #FAIL 271 270 36338 -36608
# #FAIL 271 270 36338 -36608
# return
dfn = df - dff
if plot_q:
plt.figure(figsize=(10, 4))
plt.plot(x, df, '.k', label='Orig dF')
plt.plot(x, dff1, '-c', label='dF fit 1')
plt.plot(x, dff, '-m', label='dF fit 2')
plt.legend()
for sd in [5, 3]:
dfn = sigma_clip(dfn, sigma=sd, maxiters=5)
if plot_q:
plt.figure(figsize=(10, 7))
plt.title('RMS = ' + str(np.std(dfn)))
plt.plot(x, dfn, '.k')
return np.std(dfn)
def eden_GPDetrend(telescope, datafolder, targets, calibrated=True):
# define constants from config.ini
config = ConfigParser()
config.read('config.ini')
server_destination = config['FOLDER OPTIONS']['server_destination']
# create GPDetrend inputs, and run GPDetrend
# create folder in post_processing for GPLC files
if calibrated:
out_dir = datafolder + 'calib_post_processing/' + 'GPLC/'
else:
out_dir = datafolder + 'post_processing/' + 'GPLC/'
if not os.path.isdir(out_dir):
os.mkdir(out_dir)
# Load necessary files to create GPDetrend inputs
nflux = np.genfromtxt(datafolder + 'post_processing/' + targets[0] +
'.dat')
LC = np.genfromtxt(datafolder + 'post_processing/LC/' + targets[0] +
'.epdlc')
comps = glob.glob(datafolder + 'post_processing/comp_light_curves/*')
# Store information
# lc file
times = nflux[:, 0]
flux = nflux[:, 1]
# sigma clip flux
filt = sigma_clip(flux, sigma=5)
filt = np.invert(filt.mask)
# eparams file
etime = LC[:, 1]
airmass = LC[:, 22]
FWHM = LC[:, 19]
cen_X = LC[:, 15]
cen_Y = LC[:, 16]
bg = LC[:, 17]
# for some reason the eparam times do not always match the flux times, this (usually) finds and removes the extras
if len(times) != len(airmass):
print('LC length does not match comps and eparams length!')
print('Time length: ', len(times), 'eparams length: ', len(airmass))
# rounding to 5 gets rid of small differences
times = np.round(times, decimals=5)
etime = np.round(etime, decimals=5)
mask = np.in1d(etime, times) # find values truly not in lc
airmass = airmass[mask]
FWHM = FWHM[mask]
cen_X = cen_X[mask]
cen_Y = cen_Y[mask]
bg = bg[mask]
# comps file
cflux = np.zeros((len(times), int(len(comps) / 4)))
count = 0
for j in range(len(comps)):
if 'pdf' or 'sigma' or 'norm' not in comps[j]:
try: # does not always work
comp = np.genfromtxt(comps[j])
if len(times) != len(airmass):
comp = comp[mask]
cflux[:, count] = comp[:, 1]
count = count + 1
except:
pass
else:
pass
# sigma mask
times, flux = times[filt], flux[filt]
airmass, FWHM, cen_X, cen_Y, bg = airmass[filt], FWHM[filt], cen_X[
filt], cen_Y[filt], bg[filt]
cflux = cflux[filt, :]
# Write the GPDetrend input files
# array format
light = np.array([times, flux, np.zeros(len(times))])
eparams = np.array([times, airmass, FWHM, bg, cen_X, cen_Y], dtype='float')
# the FWHM often contains nans, this removes those times from all files.
rem = np.where(np.isnan(FWHM))
# Remove times with FWHM Nans and transpose
eparams = np.delete(np.transpose(eparams), rem, axis=0)
light = np.delete(np.transpose(light), rem, axis=0)
cflux = np.delete(cflux, rem, axis=0)
# write
lfile = out_dir + 'lc.dat'
efile = out_dir + 'eparams.dat'
cfile = out_dir + 'comps.dat'
ofolder = 'detrend_'
np.savetxt(lfile, light, fmt='%1.6f', delimiter=' ')
np.savetxt(cfile, cflux, fmt='%1.6f', delimiter=' ')
np.savetxt(efile,
eparams,
fmt='%1.6f',
delimiter=' ',
header='times, airmass, fwhm, background, x_cen, y_cen')
# RUN DETREND
# changing directories seems necessary, otherwise the out folder is too long a name for multinest
mycwd = os.getcwd()
os.chdir(out_dir)
os.system('python ' + mycwd + '/GPDetrend.py -ofolder ' + ofolder +
' -lcfile ' + lfile + ' -eparamfile ' + efile + ' -compfile ' +
cfile + ' -eparamtouse all')
os.chdir(mycwd)
def centroid_FWM(image_data,
xo=[],
yo=[],
wx=[],
wy=[],
scale=1,
bounds=(13, 18, 13, 18)):
'''
Gets the centroid of the target by flux weighted mean and the PSF width
of the target.
Parameters:
-----------
img_data :(3D array)
Data cube of images (2D arrays of pixel values).
xo : list (optional)
List of x-centroid obtained previously. Default if none given is an
empty list.
yo : list (optional)
List of y-centroids obtained previously. Default if none given is an
empty list.
wx : list (optional)
List of PSF width (x-axis) obtained previously. Default if none given
is an empty list.
wy : list (optional)
List of PSF width (x-axis) obtained previously. Default if none given
is an empty list.
scale : int (optional)
If the image is oversampled, scaling factor for centroid and bounds,
i.e, give centroid in terms of the pixel value of the initial image.
bounds : tuple (optional)
Bounds of box around the target to exclude background . Default is (11, 19 ,11, 19).
Returns:
--------
xo : list
Updated list of x-centroid obtained previously.
yo : list
Updated list of y-centroids obtained previously.
wx : list
Updated list of PSF width (x-axis) obtained previously.
wy : list
Updated list of PSF width (x-axis) obtained previously.
'''
lbx, ubx, lby, uby = bounds
lbx, ubx, lby, uby = lbx * scale, ubx * scale, lby * scale, uby * scale
starbox = image_data[:, lbx:ubx, lby:uby]
h, w, l = starbox.shape
# get centroid
X, Y = np.mgrid[:w, :l]
cx = (np.sum(np.sum(X * starbox, axis=1), axis=1) /
(np.sum(np.sum(starbox, axis=1), axis=1))) + lbx
cy = (np.sum(np.sum(Y * starbox, axis=1), axis=1) /
(np.sum(np.sum(starbox, axis=1), axis=1))) + lby
cx = sigma_clip(cx, sigma=4, iters=2, cenfunc=np.ma.median)
cy = sigma_clip(cy, sigma=4, iters=2, cenfunc=np.ma.median)
xo.extend(cx / scale)
yo.extend(cy / scale)
# get PSF widths
X, Y = np.repeat(X[np.newaxis, :, :], h,
axis=0), np.repeat(Y[np.newaxis, :, :], h, axis=0)
cx, cy = np.reshape(cx, (h, 1, 1)), np.reshape(cy, (h, 1, 1))
X2, Y2 = (X + lbx - cx)**2, (Y + lby - cy)**2
widx = np.sqrt(
np.sum(np.sum(X2 * starbox, axis=1), axis=1) /
(np.sum(np.sum(starbox, axis=1), axis=1)))
widy = np.sqrt(
np.sum(np.sum(Y2 * starbox, axis=1), axis=1) /
(np.sum(np.sum(starbox, axis=1), axis=1)))
widx = sigma_clip(widx, sigma=4, iters=2, cenfunc=np.ma.median)
widy = sigma_clip(widy, sigma=4, iters=2, cenfunc=np.ma.median)
wx.extend(widx / scale)
wy.extend(widy / scale)
return xo, yo, wx, wy
campaign_no = campaign_no[:-31]
print("CAMPAIGN " + str(campaign_no))
#normalize to 0.0
flux = [i-1 for i in flux]
#print('flux = ' + str(type(flux)))
flux = np.asarray(flux)
time = np.asarray(time)
#print(type(flux))
#SIGMA CLIPPING
flux = sigma_clip(flux, sigma=3, iters=1)
#uncomment if extra time stamp
#time.pop()
period = np.random.uniform(low=1, high=26)
#period = 8.0
tested_period.append(period)
print('Inj period = ' + str(period))
#rprs = np.random.choice(np.arange(.1, .9, .1), 1)
rprs = np.random.uniform(low=.05, high=.95)
#rprs = .6
def cube_detect_badfr_ellipticity(array, fwhm, crop_size=30, roundlo=-0.2,
roundhi=0.2, plot=True, verbose=True):
""" Returns the list of bad frames from a cube by measuring the PSF
ellipticity of the central source. Should be applied on a recentered cube.
Parameters
----------
array : numpy ndarray
Input 3d array, cube.
fwhm : float
FWHM size in pixels.
crop_size : int, optional
Size in pixels of the square subframe to be analyzed.
roundlo, roundhi : float, optional
Lower and higher bounds for the ellipticity. See ``Notes`` below for
details.
plot : bool, optional
If true it plots the central PSF roundness for each frame.
verbose : bool, optional
Whether to print to stdout or not.
Returns
-------
good_index_list : numpy ndarray
1d array of good indices.
bad_index_list : numpy ndarray
1d array of bad frames indices.
Notes
-----
From photutils.DAOStarFinder documentation:
DAOFIND calculates the object roundness using two methods. The 'roundlo'
and 'roundhi' bounds are applied to both measures of roundness. The first
method ('roundness1'; called 'SROUND' in DAOFIND) is based on the source
symmetry and is the ratio of a measure of the object's bilateral (2-fold)
to four-fold symmetry. The second roundness statistic ('roundness2'; called
'GROUND' in DAOFIND) measures the ratio of the difference in the height of
the best fitting Gaussian function in x minus the best fitting Gaussian
function in y, divided by the average of the best fitting Gaussian
functions in x and y. A circular source will have a zero roundness. A source
extended in x or y will have a negative or positive roundness, respectively.
"""
from .cosmetics import cube_crop_frames
check_array(array, 3, msg='array')
if verbose:
start_time = time_ini()
array = cube_crop_frames(array, crop_size, verbose=False)
n = array.shape[0]
goodfr = []
badfr = []
roundness1 = []
roundness2 = []
for i in range(n):
ff_clipped = sigma_clip(array[i], sigma=3, maxiters=None)
thr = ff_clipped.max()
DAOFIND = DAOStarFinder(threshold=thr, fwhm=fwhm)
tbl = DAOFIND.find_stars(array[i])
table_mask = (tbl['peak'] == tbl['peak'].max())
tbl = tbl[table_mask]
roun1 = tbl['roundness1'][0]
roun2 = tbl['roundness2'][0]
roundness1.append(roun1)
roundness2.append(roun2)
# we check the roundness
if roundhi > roun1 > roundlo and roundhi > roun2 > roundlo:
goodfr.append(i)
else:
badfr.append(i)
bad_index_list = np.array(badfr)
good_index_list = np.array(goodfr)
if plot:
_, ax = plt.subplots(figsize=vip_figsize)
x = range(len(roundness1))
if n > 5000:
marker = ','
else:
marker = 'o'
ax.plot(x, roundness1, '-', alpha=0.6, color='#1f77b4',
label='roundness1')
ax.plot(x, roundness1, marker=marker, alpha=0.4, color='#1f77b4')
ax.plot(x, roundness2, '-', alpha=0.6, color='#9467bd',
label='roundness2')
ax.plot(x, roundness2, marker=marker, alpha=0.4, color='#9467bd')
ax.hlines(roundlo, xmin=-1, xmax=n + 1, lw=2, colors='#ff7f0e',
linestyles='dashed', label='roundlo', alpha=0.6)
ax.hlines(roundhi, xmin=-1, xmax=n + 1, lw=2, colors='#ff7f0e',
linestyles='dashdot', label='roundhi', alpha=0.6)
plt.xlabel('Frame number')
plt.ylabel('Roundness')
plt.xlim(xmin=-1, xmax=n + 1)
plt.legend(fancybox=True, framealpha=0.5, loc='best')
plt.grid('on', alpha=0.2)
if verbose:
bad = len(bad_index_list)
percent_bad_frames = (bad*100)/n
msg1 = "Done detecting bad frames from cube: {} out of {} ({:.3}%)"
print(msg1.format(bad, n, percent_bad_frames))
timing(start_time)
return good_index_list, bad_index_list
def test_median(median):
'''Test median image.'''
save_figs = True
# create fake data for subpixel dither 1 to test median
with fits.open("V54321001002P0000000001101_A1_F150W_cal.fits") as h:
h['SCI'].data[:,:] = 3.0
h['DQ'].data[:,:] = 0
h.writeto("V54321001002P0000000001101_A1_F150W_cal_mediantest.fits",overwrite=True)
# create fake data for subpixel dither 2 to test median
with fits.open("V54321001002P0000000001102_A1_F150W_cal.fits") as h:
h['SCI'].data[:,:] = 5.0
h['DQ'].data[:,:] = 0
h.writeto("V54321001002P0000000001102_A1_F150W_cal_mediantest.fits",overwrite=True)
# run Image3 pipeline to get outlier detection outputs
im3 = calwebb_image3.Image3Pipeline(config_file='calwebb_image3.cfg')
im3.tweakreg.skip = True
im3.resample.blendheaders = False
im3.outlier_detection.save_intermediate_results = True
im3.run(median)
# get file names and bases to load output data
input_files = []
with open(median) as json_data:
d = json.load(json_data)
members = d['products'][0]['members']
base = members[0]['expname'][1:8]
output_files = glob("jw"+base+"*01101_00001_outlier_i2d.fits") + \
glob("jw"+base+"*01102_00001_outlier_i2d.fits")
input_file_base = members[0]['expname']
# put results into array
with fits.open(output_files[0]) as h:
shape = h[1].data.shape
all_dithers = np.zeros((len(output_files),shape[0],shape[1]),dtype='float32')
for i in np.arange(0,len(all_dithers)):
print(np.shape(fits.getdata(output_files[i],1)))
all_dithers[i,:,:] = fits.getdata(output_files[i],1)
# get output median image
first = input_file_base
cal = first.find("_F")
median_file = first[:cal]+"_median.fits"
median_image = fits.getdata(median_file,1)
# manually calculate the median
calc_median_array = np.nanmedian(all_dithers,axis=0)
diff_array = np.abs(calc_median_array - median_image)
# sigma-clipping
clip = sigma_clip(diff_array)
clip.data[clip.mask] = np.nan
diff_mean = np.nanmean(clip.data)
if save_figs == True:
# save figure to show median
fig, ax = plt.subplots(1,1,figsize=(10,10))
plt.ylabel('y pixels',fontsize=15)
plt.xlabel('x pixels',fontsize=15)
plt.imshow(median_image, vmin=3.0, vmax=5.0, cmap=plt.cm.gray, origin='lower')
ax.set_title("Median image",fontsize=12)
plt.colorbar(orientation='horizontal',pad=0.09)
plt.savefig(median[:5]+"_median_image.png")
# save figure to show difference between calculated and output medians
fig, ax = plt.subplots(1,1,figsize=(10,10))
plt.ylabel('y pixels',fontsize=15)
plt.xlabel('x pixels',fontsize=15)
plt.imshow(diff_array, vmin=-0.5, vmax=0.5, cmap=plt.cm.gray, origin='lower')
ax.set_title("Difference between median image and manually calculated median",fontsize=12)
plt.colorbar(orientation='horizontal',pad=0.09)
plt.savefig(median[:5]+"_median_diff_image.png")
assert np.isclose(diff_mean,0.0) == True
def tls_search(time,
flux,
flux_err,
known_transits=None,
tls_kwargs=None,
wotan_kwargs=None,
options=None):
'''
Summary:
-------
This runs TLS on these data with the given infos
Inputs:
-------
time : array of flaot
time stamps of observations
flux : array of flaot
normalized flux
flux_err : array of flaot
error of normalized flux
Optional Inputs:
----------------
tls_kwargs : None or dict, keywords:
R_star : float
radius of the star (e.g. median)
default 1 R_sun (from TLS)
R_star_min : float
minimum radius of the star (e.g. 1st percentile)
default 0.13 R_sun (from TLS)
R_star_max : float
maximum radius of the star (e.g. 99th percentile)
default 3.5 R_sun (from TLS)
M_star : float
mass of the star (e.g. median)
default 1. M_sun (from TLS)
M_star_min : float
minimum mass of the star (e.g. 1st percentile)
default 0.1 M_sun (from TLS)
M_star_max : float
maximum mass of the star (e.g. 99th percentile)
default 1. M_sun (from TLS)
u : list
quadratic limb darkening parameters
default [0.4804, 0.1867]
...
SNR_threshold : float
the SNR threshold at which to stop the TLS search
known_transits : None or dict
if dict and one transit is already known:
known_transits = {'period':[1.3], 'duration':[2.1], 'epoch':[245800.0]}
if dict and multiple transits are already known:
known_transits = {'name':['b','c'], 'period':[1.3, 21.0], 'duration':[2.1, 4.1], 'epoch':[245800.0, 245801.0]}
'period' is the period of the transit
'duration' must be the total duration, i.e. from first ingress point to last egrees point, in days
'epoch' is the epoch of the transit
options : None or dict, keywords:
show_plot : bool
show a plot of each phase-folded transit candidate and TLS model in the terminal
default is False
save_plot : bool
save a plot of each phase-folded transit candidate and TLS model into outdir
default is False
outdir : string
if None, use the current working directory
default is ""
Returns:
-------
List of all TLS results
'''
#::: seeed
np.random.seed(42)
#::: handle inputs
if flux_err is None:
ind = np.where(~np.isnan(time * flux))[0]
time = time[ind]
flux = flux[ind]
else:
ind = np.where(~np.isnan(time * flux * flux_err))[0]
time = time[ind]
flux = flux[ind]
flux_err = flux_err[ind]
time_input = 1. * time
flux_input = 1. * flux #for plotting
if wotan_kwargs is None:
detrend = False
else:
detrend = True
if 'slide_clip' not in wotan_kwargs: wotan_kwargs['slide_clip'] = {}
if 'window_length' not in wotan_kwargs['slide_clip']:
wotan_kwargs['slide_clip']['window_length'] = 1.
if 'low' not in wotan_kwargs['slide_clip']:
wotan_kwargs['slide_clip']['low'] = 20.
if 'high' not in wotan_kwargs['slide_clip']:
wotan_kwargs['slide_clip']['high'] = 3.
if 'flatten' not in wotan_kwargs: wotan_kwargs['flatten'] = {}
if 'method' not in wotan_kwargs['flatten']:
wotan_kwargs['flatten']['method'] = 'biweight'
if 'window_length' not in wotan_kwargs['flatten']:
wotan_kwargs['flatten']['window_length'] = 1.
#the rest is filled automatically by Wotan
if tls_kwargs is None: tls_kwargs = {}
if 'show_progress_bar' not in tls_kwargs:
tls_kwargs['show_progress_bar'] = False
if 'SNR_threshold' not in tls_kwargs: tls_kwargs['SNR_threshold'] = 5.
if 'SDE_threshold' not in tls_kwargs: tls_kwargs['SDE_threshold'] = 5.
if 'FAP_threshold' not in tls_kwargs: tls_kwargs['FAP_threshold'] = 0.05
tls_kwargs_original = {
key: tls_kwargs[key]
for key in tls_kwargs.keys()
if key not in ['SNR_threshold', 'SDE_threshold', 'FAP_threshold']
} #for the original tls
#the rest is filled automatically by TLS
if options is None: options = {}
if 'show_plot' not in options: options['show_plot'] = False
if 'save_plot' not in options: options['save_plot'] = False
if 'outdir' not in options: options['outdir'] = ''
#::: init
SNR = 1e12
SDE = 1e12
FAP = 0
FOUND_SIGNAL = False
results_all = []
if len(options['outdir']) > 0 and not os.path.exists(options['outdir']):
os.makedirs(options['outdir'])
#::: logprint
with open(os.path.join(options['outdir'], 'logfile.log'), 'w') as f:
f.write('TLS search, UTC ' +
datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S') + '\n')
logprint('\nWotan kwargs:', options=options)
logpprint(wotan_kwargs, options=options)
logprint('\nTLS kwargs:', options=options)
logpprint(tls_kwargs, options=options)
logprint('\nOptions:', options=options)
logpprint(options, options=options)
#::: apply a mask (if wished so)
if known_transits is not None:
for period, duration, T0 in zip(known_transits['period'],
known_transits['duration'],
known_transits['epoch']):
time, flux, flux_err = mask(time, flux, flux_err, period, duration,
T0)
#::: global sigma clipping
flux = sigma_clip(flux, sigma_upper=3, sigma_lower=20)
#::: detrend (if wished so)
if detrend:
flux = slide_clip(time, flux, **wotan_kwargs['slide_clip'])
flux, trend = flatten(time,
flux,
return_trend=True,
**wotan_kwargs['flatten'])
if options['show_plot'] or options['save_plot']:
fig, axes = plt.subplots(2, 1, figsize=(40, 8))
axes[0].plot(time, flux_input, 'b.', rasterized=True)
axes[0].plot(time, trend, 'r-', lw=2)
axes[0].set(ylabel='Flux (input)', xticklabels=[])
axes[1].plot(time, flux, 'b.', rasterized=True)
axes[1].set(ylabel='Flux (detrended)', xlabel='Time (BJD)')
if options['save_plot']:
fig.savefig(os.path.join(
options['outdir'],
'flux_' + wotan_kwargs['flatten']['method'] + '.pdf'),
bbox_inches='tight')
if options['show_plot']:
plt.show(fig)
else:
plt.close(fig)
X = np.column_stack((time, flux, flux_err, trend))
np.savetxt(os.path.join(
options['outdir'],
'flux_' + wotan_kwargs['flatten']['method'] + '.csv'),
X,
delimiter=',',
header='time,flux_detrended,flux_err,trend')
time_detrended = 1. * time
flux_detrended = 1. * flux #for plotting
#::: search for the rest
i = 0
ind_trs = []
while (SNR >= tls_kwargs['SNR_threshold']
) and (SDE >= tls_kwargs['SDE_threshold']) and (
FAP <= tls_kwargs['FAP_threshold']) and (FOUND_SIGNAL == False):
model = tls(time, flux, flux_err)
results = model.power(**tls_kwargs_original)
if (results.snr >= tls_kwargs['SNR_threshold']) and (
results.SDE >= tls_kwargs['SDE_threshold']) and (
results.FAP <= tls_kwargs['FAP_threshold']):
#::: calculcate the correct_duration, as TLS sometimes returns unreasonable durations
ind_tr_phase = np.where(results['model_folded_model'] < 1.)[0]
correct_duration = results['period'] * (
results['model_folded_phase'][ind_tr_phase[-1]] -
results['model_folded_phase'][ind_tr_phase[0]])
#::: mark transit
ind_tr, ind_out = index_transits(time_input, results['T0'],
results['period'],
correct_duration)
ind_trs.append(ind_tr)
#::: mask out detected transits and append results
time1, flux1 = time, flux #for plotting
time, flux, flux_err = mask(time, flux, flux_err, results.period,
np.max((1.5 * correct_duration)),
results.T0)
results_all.append(results)
#::: write TLS stats to file
with open(
os.path.join(options['outdir'],
'tls_signal_' + str(i) + '.txt'),
'wt') as out:
pprint(results, stream=out)
#::: individual TLS plots
if options['show_plot'] or options['save_plot']:
fig = plt.figure(figsize=(20, 8), tight_layout=True)
gs = fig.add_gridspec(2, 3)
ax = fig.add_subplot(gs[0, :])
ax.plot(time1, flux1, 'b.', rasterized=True)
ax.plot(results['model_lightcurve_time'],
results['model_lightcurve_model'],
'r-',
lw=3)
ax.set(xlabel='Time (BJD)', ylabel='Flux')
ax = fig.add_subplot(gs[1, 0])
ax.plot(results['folded_phase'],
results['folded_y'],
'b.',
rasterized=True)
ax.plot(results['model_folded_phase'],
results['model_folded_model'],
'r-',
lw=3)
ax.set(xlabel='Phase', ylabel='Flux')
ax = fig.add_subplot(gs[1, 1])
ax.plot(
(results['folded_phase'] - 0.5) * results['period'] * 24,
results['folded_y'],
'b.',
rasterized=True)
ax.plot((results['model_folded_phase'] - 0.5) *
results['period'] * 24,
results['model_folded_model'],
'r-',
lw=3)
ax.set(xlim=[
-1.5 * correct_duration * 24, +1.5 * correct_duration * 24
],
xlabel='Time (h)',
yticks=[])
ax = fig.add_subplot(gs[1, 2])
ax.text(.02,
0.95,
'P = ' +
np.format_float_positional(results['period'], 4) +
' d',
ha='left',
va='center',
transform=ax.transAxes)
ax.text(
.02,
0.85,
'Depth = ' +
np.format_float_positional(1e3 *
(1. - results['depth']), 4) +
' ppt',
ha='left',
va='center',
transform=ax.transAxes)
ax.text(.02,
0.75,
'Duration = ' +
np.format_float_positional(24 * correct_duration, 4) +
' h',
ha='left',
va='center',
transform=ax.transAxes)
ax.text(.02,
0.65,
'T_0 = ' +
np.format_float_positional(results['T0'], 4) + ' d',
ha='left',
va='center',
transform=ax.transAxes)
ax.text(.02,
0.55,
'SNR = ' +
np.format_float_positional(results['snr'], 4),
ha='left',
va='center',
transform=ax.transAxes)
ax.text(.02,
0.45,
'SDE = ' +
np.format_float_positional(results['SDE'], 4),
ha='left',
va='center',
transform=ax.transAxes)
ax.text(.02,
0.35,
'FAP = ' +
np.format_float_scientific(results['FAP'], 4),
ha='left',
va='center',
transform=ax.transAxes)
ax.set_axis_off()
if options['save_plot']:
fig.savefig(os.path.join(options['outdir'],
'tls_signal_' + str(i) + '.pdf'),
bbox_inches='tight')
if options['show_plot']:
plt.show(fig)
else:
plt.close(fig)
SNR = results.snr
SDE = results.SDE
FAP = results.FAP
i += 1
#::: full lightcurve plot
if options['show_plot'] or options['save_plot']:
if detrend:
fig, axes = plt.subplots(2, 1, figsize=(40, 8), tight_layout=True)
ax = axes[0]
ax.plot(time_input,
flux_input,
'k.',
color='grey',
rasterized=True)
ax.plot(time_input, trend, 'r-', lw=2)
for number, ind_tr in enumerate(ind_trs):
ax.plot(time_input[ind_tr],
flux_input[ind_tr],
marker='.',
linestyle='none',
label='signal ' + str(number))
ax.set(ylabel='Flux (input)', xticklabels=[])
ax.legend()
ax = axes[1]
ax.plot(time_detrended,
flux_detrended,
'k.',
color='grey',
rasterized=True)
for number, ind_tr in enumerate(ind_trs):
ax.plot(time_detrended[ind_tr],
flux_detrended[ind_tr],
marker='.',
linestyle='none',
label='signal ' + str(number))
ax.set(ylabel='Flux (detrended)', xlabel='Time (BJD)')
ax.legend()
else:
fig = plt.figure(figsize=(20, 4), tight_layout=True)
fig, ax = plt.subplots(1, 1, figsize=(40, 4))
ax.plot(time_input,
flux_input,
'k.',
color='grey',
rasterized=True)
ax.set(ylabel='Flux (input)', xlabel='Time (BJD)')
for number, ind_tr in enumerate(ind_trs):
ax.plot(time_input[ind_tr],
flux_input[ind_tr],
marker='.',
linestyle='none',
label='signal ' + str(number))
ax.legend()
if options['save_plot']:
fig.savefig(os.path.join(options['outdir'], 'tls_signal_all.pdf'),
bbox_inches='tight')
if options['show_plot']:
plt.show(fig)
else:
plt.close(fig)
return results_all
def sens_func(wav,
cnt,
exp,
seeing,
sw,
airmass,
extfile,
stdfile,
wave_range,
order,
show=1,
save=0):
'''
'''
# Speed of light
c = 2.99792458e+10 # [cm/s]
inverse = False
if wav[1] < wav[0]:
wav = wav[::-1]
cnt = cnt[::-1]
inverse = True
# Slit loss correction
# --------------------
print(
F'[Sensitivity function] Slit loss correction: FWHM = {seeing:.2f} [px], Slit Width = {sw:.2f} [px]'
)
sl = get_slit_loss(fwhm=seeing, sw=sw)
cnt = cnt / (1 - sl)
# Atmospheric extinction correction
# ---------------------------------
print(
'[Sensitivity function] Extinction correction: Load extinction coefficients'
)
wav_ext, ext = np.loadtxt(extfile).T
print('[Sensitivity function] Extinction correction: Dereddening')
ext = interp1d(wav_ext,
ext,
kind='quadratic',
bounds_error=False,
fill_value='extrapolate')(wav)
ext_corr_factor = 10**(0.4 * airmass * ext)
cnt = cnt * ext_corr_factor
# Convert to counts/A/s
# ---------------------
# 1. Bandpass
dwav = np.abs(np.diff(wav))
dwav = np.hstack([dwav[0], dwav, dwav[-1]])
dwav = (dwav[:-1] + dwav[1:]) / 2
# 2. Convert
cnt = cnt / dwav / exp
# Sensitivity function
# --------------------
if not os.path.exists(os.path.join(os.getcwd(), stdfile)):
if not os.path.exists(
os.path.join(
os.path.split(os.path.realpath(__file__))[0], 'onedstds/',
stdfile)):
raise FileNotFoundError('No standard file found.')
sys.exit()
else:
stdfile = os.path.join(
os.path.split(os.path.realpath(__file__))[0], 'onedstds/',
stdfile)
else:
stdfile = os.path.join(os.getcwd(), stdfile)
# 1. Load standard spectrum
print('[Sensitivity function] Load archived standard spectrum')
if os.path.split(os.path.split(stdfile)[0])[1] == 'calspec':
tab = Table.read(stdfile)
wav_mag = tab['WAVELENGTH'].data
flx_mod = tab['FLUX'].data
bp = tab['FWHM'].data
else:
stdspec = np.loadtxt(stdfile, skiprows=1).T
if stdspec.shape[0] == 3:
wav_mag, mag, bp = stdspec
elif stdspec.shape[0] == 2:
wav_mag, mag = stdspec
dwav_mag = np.abs(np.diff(wav_mag))
dwav_mag = np.hstack([dwav_mag[0], dwav_mag, dwav_mag[-1]])
bp = (dwav_mag[:-1] + dwav_mag[1:]) / 2
flx_mod = 10**(-0.4 *
mag) * 3631e-23 * c / wav_mag**2 * 1e8 # [erg/cm2/s/A]
# 2. Comparision
print('[Sensitivity function] Comparision')
flx_obs = np.zeros(flx_mod.shape[0])
bins = np.vstack([wav_mag - bp / 2, wav_mag + bp / 2]).T
for i, bin in enumerate(bins):
idx = (bin[0] < wav) & (wav < bin[1])
edges = interp1d(wav,
cnt,
'linear',
bounds_error=False,
fill_value=np.nan)(bin)
flx_obs[i] = np.trapz(np.hstack([edges[0], cnt[idx], edges[1]]),
x=np.hstack([bin[0], wav[idx], bin[1]
])) / (bin[1] - bin[0])
sen = flx_obs / flx_mod
# 3. Filter NaN
idx = np.isnan(sen)
wav_sen = wav_mag[~idx]
sen = 2.5 * np.log10(sen[~idx])
print('[Sensitivity function] Fitting')
mask = ~np.isnan(sen) & (wave_range[0] < wav_sen) & (wav_sen <
wave_range[1])
for i in range(5):
knots = np.r_[(wav_sen[mask][0], ) * (order + 1),
(wav_sen[mask][-1], ) * (order + 1)]
spl = make_lsq_spline(wav_sen[mask], sen[mask], t=knots, k=order)
mask = mask & ~sigma_clip(
sen - spl(wav_sen), sigma=0.5, maxiters=1, masked=True).mask
sen_fit = spl(wav)
# Plot
print('[Sensitivity function] Plot fitted sensitivity function', end='')
fig, ax = plt.subplots(2, 1, figsize=(12, 12))
# Sensitivity function
ax[0].plot(wav_sen[mask], sen[mask], '+', color='black', ms=10)
ax[0].plot(wav_sen[~mask], sen[~mask], '+', color='lightgrey', ms=10)
ax[0].plot(wav, sen_fit, '-', c='red', lw=2)
# Settings
ax[0].grid(axis='both', color='0.95', zorder=0)
ax[0].set_xlim(wav.min(), wav.max())
ax[0].tick_params(which='major',
direction='in',
top=True,
right=True,
length=7,
width=1.5,
labelsize=18)
ax[0].set_ylabel('Sensitivity Function', fontsize=22)
ax[0].set_title('Sensitivity Function', fontsize=24)
ax[1].plot(wav_sen[mask], (sen - spl(wav_sen))[mask],
'+',
color='black',
ms=10,
zorder=1)
ax[1].plot(wav_sen[~mask], (sen - spl(wav_sen))[~mask],
'+',
color='lightgrey',
ms=10,
zorder=0)
ax[1].axhline(y=0, c='red', ls='--', zorder=2)
# Settings
ax[1].grid(axis='both', color='0.95')
ax[1].set_xlim(wav.min(), wav.max())
ax[1].set_ylim((sen - spl(wav_sen))[mask].min() * 0.8,
(sen - spl(wav_sen))[mask].max() * 1.2)
ax[1].tick_params(which='major',
direction='in',
top=True,
right=True,
length=7,
width=1.5,
labelsize=18)
ax[1].set_xlabel('Wavelength [$\\mathrm{\\AA}$]', fontsize=22)
ax[1].set_ylabel('Residuals', fontsize=22)
fig.align_ylabels()
if save:
print(' to `figs/Sensitivity_Function.png`')
plt.savefig('figs/Sensitivity_Function.png', dpi=144)
else:
print('')
if show:
print('[Sensitivity function] Show plot')
plt.show()
plt.close()
if inverse:
sen_fit = sen_fit[::-1]
wav = wav[::-1]
cnt = cnt[::-1]
# Write to file
if not os.path.exists('cal'): os.makedirs('cal')
file_path = F'cal/SensFunc.dat'
print(
F'[Sensitivity function] Write sensitivity function to `{file_path}`')
np.savetxt(file_path, np.vstack([wav, sen_fit]).T, fmt='%15.8e')
return sen_fit
#%%
bias = combine(bias_list, # ccdproc does not accept numpy.ndarray, but only python list.
method='median', # default is average so I specified median.
unit='adu') # unit is required: it's ADU in our case.
#print('bias', bias.data.dtype, bias.data.max(), bias.data.min(), bias )
bias.write(dir_name+f_name_bias, overwrite =True)
#%%
dark0 = combine(dark_list, # ccdproc does not accept numpy.ndarray, but only python list.
method='median', # default is average so I specified median.
unit='adu')
dark0.write(dir_name+f_name_dark0, overwrite =True)
print('dark0', dark0.data.min(), dark0.data.max(), dark0)
dark = ccd_process(dark0, master_bias=bias)
print('dark', dark.data.min(), dark.data.max(), dark.data)
dark_clip = sigma_clip(dark)
print('dark_clip', dark_clip.data.min(), dark_clip.data.max(), dark_clip.data)
dark_clip.fill_value = np.median(dark_clip.data)
dark.data = dark_clip.filled() # ~.filled() gives the "data array using filled_value"
print('dark', dark.data.dtype, dark.data.min(), dark.data.max(), dark.data)
#dark.write(dir_name+f_name_dark, overwrite =True)
#%%
for chl in[''] :
flat_list = sorted(glob(os.path.join(dir_name, 'flat-'+chl+'*.fit')))
flat0 = combine(flat_list, # ccdproc does not accept numpy.ndarray, but only python list.
method='median', # default is average so I specified median.
unit='adu')
flat0.write(dir_name+f_name_flat0[:-4]+'_'+chl+'.fit', overwrite =True)
def find_resolution(multispec_fname,
initial_fwhm=.05,
usepercentile=True,
percentiles=[60, 80, 95],
Rguess=None,
full_output=False,
useclip=True,
findpeak=False,
makeplot=True):
"""
"""
from .spectrum import Spectrum1D
from astropy.stats import sigma_clip, biweight
from ..robust_polyfit import gaussfit
from ..utils import find_distribution_peak
import time
arcs = Spectrum1D.read(multispec_fname, flux_ext=4)
line_centers = np.loadtxt(os.path.dirname(__file__) +
"/../data/linelists/thar_list",
usecols=0)
start = time.time()
alllinefits = []
allRmed = []
allRerr = []
allwmid = []
allR = []
for i, arc in enumerate(arcs):
linefits = []
wave = arc.dispersion
flux = arc.flux
wmin, wmax = wave[0], wave[-1]
wmid = (wmin + wmax) / 2.
lcs = line_centers[(line_centers > wmin) & (line_centers < wmax)]
for lc in lcs:
fwhm = initial_fwhm
# get subpiece of arc
ii = (wave > lc - 5 * fwhm) & (wave < lc + 5 * fwhm)
_x, _y = wave[ii], flux[ii]
# guess amplitude, center, sigma
p0 = [np.max(_y), lc, fwhm / 2.355]
try:
popt = gaussfit(_x, _y, p0)
except:
pass
else:
if popt[0] > 0 and abs(popt[1] - lc) < .05:
linefits.append(popt)
try:
A, w, s = np.array(linefits).T
except ValueError:
print("This order did not have any good lines I guess")
#allR.append(np.nan); allRmed.append(np.nan); allRerr.append(np.nan); allwmid.append(wmid)
continue
alllinefits.append(linefits)
R = w / (s * 2.355)
if useclip: R = sigma_clip(R)
if findpeak:
if Rguess is None: Rguess = np.nanmedian(R)
try:
Rmed = find_distribution_peak(R, Rguess)
except (ValueError, RuntimeError):
print("--Could not find peak for arc {:02}".format(i))
print("--{}".format(sys.exc_info()))
Rmed = np.median(R)
elif usepercentile:
assert len(percentiles) == 3
Rlo, Rmed, Rhi = np.percentile(R, percentiles)
#Rerr = max(Rhi-Rmed, Rmed-Rlo)
Rerr = (Rmed - Rlo, Rhi - Rmed)
else:
Rmed = biweight.biweight_location(R)
Rerr = biweight.biweight_scale(R)
allR.append(R)
allRmed.append(Rmed)
allRerr.append(Rerr)
allwmid.append(wmid)
if usepercentile:
allRerr = np.array(allRerr).T
if makeplot:
import matplotlib.pyplot as plt
fig, ax = plt.subplots()
ax.errorbar(allwmid, allRmed, yerr=allRerr, fmt='o')
plt.show()
if full_output:
return allRmed, allRerr, allwmid, allR, arcs, alllinefits
return allRmed, allRerr, allwmid
def avsigc(indata, sigma):
a = stats.sigma_clip(indata, sigma=sigma, iters=3)
mn = np.median(a[-a.mask])
lena = np.ma.MaskedArray.count(a)
return mn, lena
def compare_results(file_list, parameter_change_list, bary_starname, orbital_parameters_mult, objects, servaldir, serval_T0_offset):
"""
Compares result files to find the best rv scatter around literature fit returns change in parameter that yielded best results
file_list: list of `str`
list containing the file paths of the files to be compared
parameter_change_list: list of `int`
list containing the parameter exponent shifts used to create the files in file_list
orbital_parameters_mult : list of `float`
orbital_parameters = [K, P, e, omega, T0]
parameters of the keplerian fit to be used as "true" baseline
"""
def keplarian_rv_mult(t):
#Calculates keplarian rv for multiple supplied sets of planet parameters
total_rv = 0
for parameters in orbital_parameters_mult:
total_rv = total_rv + rv.radial_velocity_M0(t , parameters[0], parameters[1], parameters[2], parameters[3], parameters[4], parameters[5])
return total_rv
def fit_func(t, T0_offset):
return keplarian_rv_mult(t + T0_offset)
sigma_list = np.zeros(len(file_list)) + 100 # 100 is a fudge factor
plots = True
if plots:
rec_loop_directory, key_name = os.path.split(os.path.split(file_list[0])[0]) # HACK go up 2 directories to loop directory
plot_directory = rec_loop_directory + "/compare_plots"
os.makedirs(plot_directory, exist_ok = True)
pp =PdfPages(plot_directory +"/"+ key_name +".pdf")
fig = plt.figure(figsize=(15, 9), dpi=200)
mpl.rc('font', size=16)
plt.clf()
fig.clf()
ax1=plt.gca()
for f, fil in enumerate(file_list):
res = Results_ws(wobble_file = fil
, serval_dir = servaldir
, carmenes_object_ID = objects[0][1]
, bary_starname = bary_starname
, load_bary = False
, archive = False)
res.apply_corrections()
w_RVs_barycorr = res.w_RVs_barycorr
w_dates = res.w_dates
w_RVs_er = res.w_RVs_er
#skip fitting test RV curve to wobble data. Just use SERVAL T0 offset
T0_offset = serval_T0_offset
sigma_wob = np.nanstd(sigma_clip(
w_RVs_barycorr - np.nanmean(w_RVs_barycorr) - fit_func(w_dates, T0_offset)
,sigma = 5))
sigma_list[f] = sigma_wob
sigma_wob_noclip = np.nanstd(
w_RVs_barycorr - np.nanmean(w_RVs_barycorr) - fit_func(w_dates, T0_offset)
)
if plots:
T0_offset_s = T0_offset
ser_avcn = res.ser_avcn
sigma_ser = np.nanstd(sigma_clip(
ser_avcn[:,1] - np.nanmean(ser_avcn[:,1]) - fit_func(ser_avcn[:,0], T0_offset_s)
,sigma = 5) )
sigma_ser_noclip = np.nanstd(
ser_avcn[:,1] - np.nanmean(ser_avcn[:,1]) - fit_func(ser_avcn[:,0], T0_offset_s)
)
sigma_wob_Soffset = np.nanstd(sigma_clip(
w_RVs_barycorr - np.nanmean(w_RVs_barycorr) - fit_func(w_dates, T0_offset_s)
,sigma = 5))
sigma_list[f] = sigma_wob
sigma_wob_noclip_Soffset = np.nanstd(
w_RVs_barycorr - np.nanmean(w_RVs_barycorr) - fit_func(w_dates, T0_offset_s)
)
xlst = np.linspace(w_dates[0], w_dates[0] + orbital_parameters_mult[0][1]*0.99999, num=100)
ylst = [fit_func(t, T0_offset) for t in xlst]
#sort by xlst
pltlst = [[xlst[j],ylst[j]] for j in range(len(xlst))]
def mod_sort(elem):
return elem[0] % orbital_parameters_mult[0][1]
pltlst = sorted(pltlst, key = mod_sort)
pltlst = np.asarray(pltlst)
pltlst = [pltlst[:,0],pltlst[:,1]]
T0_source = "SERVAL "
ax1.plot(pltlst[0] % orbital_parameters_mult[0][1], pltlst[1], "r-", label = "literature orbit (" + T0_source + "T0_offset)")
ax1.errorbar((w_dates) % orbital_parameters_mult[0][1], (w_RVs_barycorr-np.nanmean(w_RVs_barycorr)), yerr = w_RVs_er,fmt = "x", label="Wobble_Corr, clipped_sigma = {0:.3f}, noclip = {1:.3f} ".format(sigma_wob, sigma_wob_noclip))
ax1.errorbar((ser_avcn[:,0]) % orbital_parameters_mult[0][1], ser_avcn[:,1] - np.nanmean(ser_avcn[:,1]),yerr = ser_avcn[:,2] ,fmt = "x", label= "SERVAL_Corr, clipped_sigma = {0:.3f}, noclip = {1:.3f}".format(sigma_ser, sigma_ser_noclip), color = "C2")
ax1.plot([], [], ' ', label="Wobble_Corr_SERVAL_fit, clipped_sigma = {0:.3f}, noclip = {1:.3f} ".format(sigma_wob_Soffset, sigma_wob_noclip_Soffset))
ax1.set_ylabel("RVs [m/s]")
ax1.set_xlabel('jd')
# add the parameter change to the title
title_pre = os.path.split(os.path.split(fil)[0])[1]
plt.title(title_pre + ", Phased ("+str(orbital_parameters_mult[0][1])+"d) RVs for "+objects[0][0]+" ("+objects[0][2]+") "+" - "+objects[0][1]+";")
plt.grid(True)
plt.tight_layout()
plt.legend(shadow=True)
plt.savefig(pp, format='pdf')
plt.clf()
fig.clf()
ax1 = plt.gca()
if plots:# include some nice progress plots. TODO make it not crudely placed inside this function?
plt.close(fig)
pp.close()
best_index = np.argmin(sigma_list)
return parameter_change_list[best_index]
def correct(self,
time,
flux,
centroid_col,
centroid_row,
polyorder=5,
niters=3,
bins=15,
windows=1,
sigma_1=3.,
sigma_2=5.,
restore_trend=False):
"""Returns a systematics-corrected LightCurve.
Note that it is assumed that time and flux do not contain NaNs.
Parameters
----------
time : array-like
Time measurements
flux : array-like
Data flux for every time point
centroid_col, centroid_row : array-like, array-like
Centroid column and row coordinates as a function of time
polyorder : int
Degree of the polynomial which will be used to fit one
centroid as a function of the other.
niters : int
Number of iterations of the aforementioned algorithm.
bins : int
Number of bins to be used in step (6) to create the
piece-wise interpolation of arclength vs flux correction.
windows : int
Number of windows to subdivide the data. The SFF algorithm
is ran independently in each window.
sigma_1, sigma_2 : float, float
Sigma values which will be used to reject outliers
in steps (6) and (2), respectivelly.
restore_trend : bool
If `True`, the long-term trend will be added back into the
lightcurve.
Returns
-------
corrected_lightcurve : LightCurve object
Returns a corrected lightcurve object.
"""
timecopy = time
time = np.array_split(time, windows)
flux = np.array_split(flux, windows)
centroid_col = np.array_split(centroid_col, windows)
centroid_row = np.array_split(centroid_row, windows)
flux_hat = np.array([])
# The SFF algorithm is going to be run on each window independently
for i in tqdm(range(windows)):
# To make it easier (and more numerically stable) to fit a
# characteristic polynomial that describes the spacecraft motion,
# we rotate the centroids to a new coordinate frame in which
# the dominant direction of motion is aligned with the x-axis.
self.rot_col, self.rot_row = self.rotate_centroids(
centroid_col[i], centroid_row[i])
# Next, we fit the motion polynomial after removing outliers
self.outlier_cent = sigma_clip(data=self.rot_col,
sigma=sigma_2).mask
with warnings.catch_warnings():
# ignore warning messages related to polyfit being poorly conditioned
warnings.simplefilter("ignore", category=np.RankWarning)
coeffs = np.polyfit(self.rot_row[~self.outlier_cent],
self.rot_col[~self.outlier_cent],
polyorder)
self.poly = np.poly1d(coeffs)
self.polyprime = np.poly1d(coeffs).deriv()
# Compute the arclength s. It is the length of the polynomial
# (fitted above) that describes the typical motion.
x = np.linspace(np.min(self.rot_row[~self.outlier_cent]),
np.max(self.rot_row[~self.outlier_cent]), 10000)
self.s = np.array(
[self.arclength(x1=xp, x=x) for xp in self.rot_row])
# Next, we find and apply the correction iteratively
self.trend = np.ones(len(time[i]))
for n in range(niters):
# First, fit a spline to capture the long-term varation
# We don't want to fit the long-term trend because we know
# that the K2 motion noise is a high-frequency effect.
self.bspline = self.fit_bspline(time[i], flux[i])
# Remove the long-term variation by dividing the flux by the spline
iter_trend = self.bspline(time[i] - time[i][0])
self.normflux = flux[i] / iter_trend
self.trend *= iter_trend
# Bin and interpolate normalized flux to capture the dependency
# of the flux as a function of arclength
self.interp = self.bin_and_interpolate(self.s,
self.normflux,
bins,
sigma=sigma_1)
# Correct the raw flux
corrected_flux = self.normflux / self.interp(self.s)
flux[i] = corrected_flux
if restore_trend:
flux[i] *= self.trend
flux_hat = np.append(flux_hat, flux[i])
return LightCurve(time=timecopy, flux=flux_hat)
def get_zp_shift( self, order, show, save ):
'''
A method to get zeropoint shift along dispersion axis.
Parameters
----------
order : int
Order used in polyfit.
show : bool
If `True`, the plot will be shown.
save : bool
If `True`, the plot will be written to file.
Returns
-------
self.shift : array_like
Fitted zeropoint shift.
'''
# Derive zeropoint shift
# ----------------------
shift = np.array([])
for k, img in enumerate( self.imgs ):
if self.slit_along == 'col': data = img * 1
if self.slit_along == 'row': data = img.T
# Find peaks
# ----------
print( F'[Zeropoint correction] Seeking for peaks in the {k+1}/{self.num} images' )
peaks, properties = find_peaks( data.mean( axis = 0 ) / data.mean( axis = 0 ).max(),
height = ( 0.3, 0.8 ),
distance = data.shape[1] // 10,
width = 0 )
# Print peak info.
tab = PrettyTable( hrules = HEADER, vrules = NONE )
tab.field_names = [ 'CENTER', 'HEIGHT', 'WIDTH' ]
for i, peak in enumerate( peaks ):
tab.add_row([ peak, round( properties['peak_heights'][i], 2 ), int( properties['widths'][i] ) ])
print( '\n' + tab.get_string() + '\n' )
# Gaussian fitting
x = np.arange( data.shape[1] )
mu = np.zeros([ data.shape[0], len( peaks ) ])
for i in range( data.shape[0] ):
print( F'\r[Zeropoint correction] Peak center fitting in the ({i+1}/{data.shape[0]})-th row of {k+1}/{self.imgs.shape[0]} images', end = '', flush = True )
for j, peak in enumerate( peaks ):
idxmin, idxmax = peak - int( properties['widths'][j] ), peak + int( properties['widths'][j] )
popt, pcov = curve_fit( Gaussian,
x[idxmin:idxmax],
data[i, idxmin:idxmax],
bounds = ( [data[i, idxmin:idxmax].max()*0.5, x[idxmin:idxmax].min(), 0 ],
[data[i, idxmin:idxmax].max()*1.5, x[idxmin:idxmax].max(), x[idxmin:idxmax].shape[0] ] ) )
mu[i, j] = popt[1]
# Convert to relative shift
shift = np.hstack([ shift.reshape( mu.shape[0], -1 ), ( mu - mu[mu.shape[0]//2] ) ])
print( '' )
# Mean
shift_mean = shift.mean( axis = 1 )
# Polyfit to shift curve
# ----------------------
print( F'[Zeropoint correction] Fitting zeropoint shift iteratively (order = {order}, 5 iterations, nsigma = 1)' )
y = np.arange( shift_mean.shape[0] ) + 1
mask = np.ones( shift_mean.shape[0], dtype = bool )
for k in range( 5 ):
knots = np.r_[ ( y[mask][0], ) * ( order + 1 ), ( y[mask][-1], ) * ( order + 1 ) ]
spl = make_lsq_spline( y[mask], shift_mean[mask], t = knots, k = order )
mask = mask & ~sigma_clip( shift_mean - spl( y ), sigma = 1, maxiters = 1, masked = True ).mask
# p = np.poly1d( np.polyfit( y[mask], shift_mean[mask], order ) )
# shift_fit = p( y )
# mask = mask & ( ~sigma_clip( shift_mean - shift_fit, sigma = 1, maxiters = 1, masked = True ).mask )
self.shift = spl( y )
# Write to file
# -------------
if not os.path.exists( 'bak' ): os.makedirs( 'bak' )
np.savetxt( 'bak/zp_shift.dat', self.shift, fmt = '%15.8e' )
# Plot
# ----
fig, ax = plt.subplots( 2, 1, figsize = ( 10, 8 ) )
fig.subplots_adjust( hspace = 0 )
# Shifts
for i in range( shift.shape[1] ):
ax[0].plot( y, shift[:, i], 'r+' )
ax[0].plot( y[~mask], shift_mean[~mask], '+', c = 'grey' )
ax[0].plot( y[mask], shift_mean[mask], 'k+' )
# Fitted shifts
ax[0].plot( self.shift, '-', c = 'yellow', lw = 2 )
# Settings
ax[0].set_xlim( y.min(), y.max() )
ax[0].tick_params( which = 'major', direction = 'in', top = True, right = True, length = 7, width = 1.5, labelsize = 18 )
ax[0].set_xticklabels([])
ax[0].set_ylabel( 'Displacement [px]', fontsize = 22 )
ax[0].set_title( 'Zeropoint Shift Curve Fitting', fontsize = 24 )
# Residuals
ax[1].plot( y[mask], shift_mean[mask] - self.shift[mask], 'k+' )
ax[1].plot( y[~mask], shift_mean[~mask] - self.shift[~mask], '+', c = 'grey' )
# Settings
ax[1].axhline( y = 0, ls = '--', c = 'yellow', lw = 2 )
ax[1].set_xlim( y.min(), y.max() )
ax[1].tick_params( which = 'major', direction = 'in', top = True, right = True, length = 7, width = 1.5, labelsize = 18 )
ax[1].set_xlabel( 'Slit', fontsize = 22 )
ax[1].set_ylabel( 'Residuals [px]', fontsize = 22 )
fig.align_ylabels()
if save:
fig_path = 'figs'
print( F'[Zeropoint correction] Plot zeropoint shift curve fitting to `{ os.path.join( fig_path, "Zeropoint_shift_fitting.png" ) }`' )
plt.savefig( os.path.join( fig_path, 'Zeropoint_shift_curve_fitting.png' ), dpi = 144 )
if show:
print( F'[Zeropoint correction] Show plot' )
plt.show()
plt.close()
return deepcopy( self.shift )
def make_fov_image(fov, pngfn=None, **kwargs):
stretch = kwargs.get('stretch', 'linear')
interval = kwargs.get('interval', 'zscale')
imrange = kwargs.get('imrange')
contrast = kwargs.get('contrast', 0.25)
ccdplotorder = ['CCD2', 'CCD4', 'CCD1', 'CCD3']
if interval == 'rms':
try:
losig, hisig = imrange
except:
losig, hisig = (2.5, 5.0)
#
cmap = kwargs.get('cmap', 'viridis')
cmap = plt.get_cmap(cmap)
cmap.set_bad('w', 1.0)
w = 0.4575
h = 0.455
rc('text', usetex=False)
fig = plt.figure(figsize=(6, 6.5))
cax = fig.add_axes([0.1, 0.04, 0.8, 0.01])
ims = [fov[ccd]['im'] for ccd in ccdplotorder]
allpix = np.ma.array(ims).flatten()
stretch = {
'linear': vis.LinearStretch(),
'histeq': vis.HistEqStretch(allpix),
'asinh': vis.AsinhStretch(),
}[stretch]
if interval == 'zscale':
iv = vis.ZScaleInterval(contrast=contrast)
vmin, vmax = iv.get_limits(allpix)
elif interval == 'rms':
nsample = 1000 // nbin
background = sigma_clip(allpix[::nsample], iters=3, sigma=2.2)
m, s = background.mean(), background.std()
vmin, vmax = m - losig * s, m + hisig * s
elif interval == 'fixed':
vmin, vmax = imrange
else:
raise ValueError
norm = ImageNormalize(vmin=vmin, vmax=vmax, stretch=stretch)
for n, (im, ccd) in enumerate(zip(ims, ccdplotorder)):
if im.ndim == 3:
im = im.mean(axis=-1)
x = fov[ccd]['x']
y = fov[ccd]['y']
i = n % 2
j = n // 2
pos = [0.0225 + i * w + i * 0.04, 0.05 + j * h + j * 0.005, w, h]
ax = fig.add_axes(pos)
_im = ax.imshow(im,
origin='lower',
extent=[x[0, 0], x[0, -1], y[0, 0], y[-1, 0]],
norm=norm,
cmap=cmap,
interpolation=kwargs.get('interpolation', 'nearest'))
if fov['coordsys'] == 'sky':
ax.set_xlim(x.max(), x.min())
else:
ax.set_xlim(x.min(), x.max())
ax.set_ylim(y.min(), y.max())
ax.xaxis.set_visible(False)
ax.yaxis.set_visible(False)
if n == 0:
cb = fig.colorbar(_im, cax, orientation='horizontal')
cb.ax.tick_params(labelsize=9)
tstr = fov.get('file', '') + ' ' + fov.get('objname', '')
title = kwargs.get('title', tstr)
title = title[-60:]
fig.text(0.5, 0.99, title, ha='center', va='top', size=12)
if pngfn is not None:
plt.savefig(pngfn)
plt.close(fig)
def plot_stats(ax,
datas,
percentiles=[1, 99],
N_extrema=None,
scale='log',
**kwargs):
''' Plot statistical summary similar but not identical to Box plots.
If you do Boxplot from matplotlib for 10 MB data, say, there can be too many
outliers to plot on the graph which makes the plotting to take a lot of
time.
Example
-------
>>> bias = CCDData.read("bias_bin11.fits")
>>> dark_1s = CCDData.read("pdark_300s_27C_bin11.fits").data / 300
>>> percentiles = [0.1, 1, 5, 95, 99, 99.9]
>>> f, ax = plt.subplots(1, 1, figsize=(8,8))
>>> ax, clipped_stats = stats.plot_stats(ax, [bias, dark_1s], percentiles=percentiles, N_extrema=5)
>>> ax.set_ylabel("ADU")
>>> ax.set_xticks(np.arange(4))
>>> ax.set_xticklabels(['', 'Bias', 'Dark_1s', ''])
>>> plt.tight_layout()
>>> plt.savefig('bias_dark_analysis.png')
'''
datas = np.atleast_1d(datas)
clipped_stats = []
for i, data in enumerate(datas):
x = i + 1
s = give_stats(data=data, percentiles=percentiles, N_extrema=N_extrema)
data_clipped = sigma_clip(data, **kwargs)
n_rej = np.count_nonzero(data_clipped.mask)
c_avg = np.mean(data_clipped)
c_med = np.median(data_clipped)
c_std = np.std(data_clipped, ddof=1)
clipped_stats.append([c_avg, c_med, c_std])
# avg, med, std from sigma-clipped data
ax.errorbar(x,
c_avg,
yerr=c_std,
marker='_',
capsize=10,
markersize=10,
label=f"sig-clipped:\nN = {s['N'] - n_rej}")
ax.scatter(x, c_med, color='r', marker='x')
# percentiles, extrema, and zscale from original data
ax.scatter([x - 0.1] * len(percentiles),
s["percentiles"],
color='k',
marker='>')
ax.scatter([x + 0.1] * 2, [s["zmin"], s["zmax"]],
color='r',
marker='<')
ax.plot(x, s["max"], color='k', marker='_', ms=20)
ax.plot(x, s["min"], color='k', marker='_', ms=20)
if N_extrema is not None:
ax.scatter([x] * N_extrema, s["ext_lo"], color='k', marker='x')
ax.scatter([x] * N_extrema, s["ext_hi"], color='k', marker='x')
ax.set_xlim(0, len(datas) + 1)
ax.set_yscale(scale)
ax.legend()
return ax, clipped_stats
print(len(aligned_images), 'images in channel', img_chl,
'are aligned')
#combine image using algned_images:
mean_image = np.mean(aligned_images,
axis=0).astype(dtype=np.uint16)
cv2.imwrite(dir_name + '/mean_image_' + img_chl + '.png',
mean_image)
print('Succeed in combining', len(imgs_to_align),
'images on channel', img_chl, '(average)')
median_image = np.median(aligned_images,
axis=0).astype(dtype=np.uint16)
cv2.imwrite(dir_name + '/median_image_' + img_chl + '.png',
median_image)
print('Succeed in combining', len(imgs_to_align),
'images on channel', img_chl, '(median)')
sigma_clip_image = sigma_clip(aligned_images, sigma=3, \
sigma_lower=None, sigma_upper=None, iters=5, axis=None, copy=True)
cv2.imwrite(dir_name + '/sigma_clip_image_' + img_chl + '.png',
sigma_clip_image[0])
print('Succeed in combining', len(imgs_to_align),
'images on channel', img_chl, '(sigma clip)')
except Exception as err:
print('Error messgae .......')
print(err)
else:
print('There is no images for alignment')
def norm_spectrum(spec, median_window=3, order=3):
'''
Normalize a spectrum
Parameters:
-----------
spec: specutils.Spectrum1D
Spectrum to normalize
median_window: int()
Window in Pixel used in median smoothing
order: int()
Order of the polynomial used to find the continuum
Returns:
--------
norm_spec: specutils.Spectrum1D
Normalized spectrum
'''
# Regions that should not be used for continuum estimation,
# such as broad atmospheric absorption bands
exclude_regions = [
SpectralRegion(4295. * u.AA, 4315. * u.AA),
#SpectralRegion(6860.* u.AA, 6880.* u.AA),
SpectralRegion(6860. * u.AA, 6910. * u.AA),
#SpectralRegion(7590.* u.AA, 7650.* u.AA),
SpectralRegion(7590. * u.AA, 7680. * u.AA),
SpectralRegion(9260. * u.AA, 9420. * u.AA),
#SpectralRegion(11100.* u.AA, 11450.* u.AA),
#SpectralRegion(13300.* u.AA, 14500.* u.AA),
]
# First estimate of the continuum
# -> will be two for late type stars because of the many absorption lines
# -> to limit execution time use simple LinearLSQFitter()
# -> reduces normalization accuracy
_cont = fit_generic_continuum(
spec,
model=models.Chebyshev1D(order),
fitter=fitting.LinearLSQFitter(),
median_window=median_window,
exclude_regions=exclude_regions,
)(spec.spectral_axis)
# Normalize spectrum
norm_spec = spec / _cont
# Sigma clip the normalized spectrum to rm spectral lines
clip_flux = sigma_clip(
norm_spec.flux,
sigma_lower=1.25,
sigma_upper=3.,
axis=0,
grow=1.,
)
# Calculate mask
mask = np.invert(clip_flux.recordmask)
# Make new spectrum
spec_mask = Spectrum1D(
spectral_axis=spec.spectral_axis[mask],
flux=spec.flux[mask],
)
# Determine new continuum
_cont = fit_generic_continuum(
spec_mask,
model=models.Chebyshev1D(order),
fitter=fitting.LinearLSQFitter(),
median_window=median_window,
exclude_regions=exclude_regions,
)(spec.spectral_axis)
# Normalize spectrum again
norm_spec = spec / _cont
return norm_spec, mad_std(norm_spec.flux)
def bias_checks(bias, overscan=False):
i = 0
rv = np.zeros(1,
dtype=[('fileName', 'S35'), ('sliceMeanAdu', 'f4', (16, )),
('sliceRmsAdu', 'f4', (16, )),
('sliceRangeAdu', 'f4', (16, )),
('dropFlag', 'i4', (16, )),
('residualMeanAdu', 'f4', (16, )),
('residualRmsAdu', 'f4', (16, ))])
fn = os.path.basename(bias).replace('.fz', '').replace('.fits', '')
print 'checking ', fn
rv['fileName'][i] = fn
fits = _open_fits(bias)
if len(fits[1:]) != 16:
print 'ERROR: %s has %d img extensions' % (fn, len(fits[1:]))
return rv
for j, hdu in enumerate(fits[1:]):
imNum = 'IM%d' % ampOrder[j]
try:
data = hdu.read().astype(np.float32)
hdr = hdu.read_header()
except:
print 'ERROR: failed to read %s[%d]' % (fn, j + 1)
continue
if overscan:
_data, oscan_cols, oscan_rows = extract_overscan(data, hdr)
#colbias = fit_overscan(oscan_cols,**kwargs)
if oscan_rows is not None:
rowbias = fit_overscan(oscan_rows,
along='rows',
method='cubic_spline')
cslice = sigma_clip(data[-22:-2:, 2:-22],
iters=1,
sigma=3.0,
axis=0)
else:
cslice = None # No row overscan to check
bottomslice = data[5:10, -16:-2].mean(axis=0)
middleslice = data[100:110, -16:-2].mean(axis=0)
else:
cslice = sigma_clip(data[1032:1048, 2:-22],
iters=1,
sigma=3.0,
axis=0)
bottomslice = data[5:10, 1000:1014].mean(axis=0)
middleslice = data[100:110, 1000:1014].mean(axis=0)
if cslice is not None:
cslice = cslice.mean(axis=0)
cslice = gaussian_filter(cslice, 17)
rv['sliceMeanAdu'][i, j] = cslice.mean()
rv['sliceRmsAdu'][i, j] = cslice.std()
rv['sliceRangeAdu'][i, j] = cslice.max() - cslice.min()
if np.median(middleslice - bottomslice) > 15:
print 'found drop in ', bias, j
rv['dropFlag'][i, j] = 1
bias_residual = overscan_subtract(data, hdr)
s = stats_region('amp_central_quadrant')
mn, sd = array_stats(bias_residual[s],
method='mean',
rms=True,
clip_sig=5.0,
clip_iters=2)
rv['residualMeanAdu'][i, j] = mn
rv['residualRmsAdu'][i, j] = sd
return rv
spectrum_id = read["spectrum_id"]
order = read["order"]
# array that specifies if a pixel is already covered.
# to start, it should be all False
covered = np.zeros((len(wl),), dtype='bool')
# #average all of the spectra in the deque together
# residual_array = np.array(self.resid_deque)
# if len(self.resid_deque) == 0:
# raise RuntimeError("No residual spectra stored yet.")
# else:
# residuals = np.average(residual_array, axis=0)
# run the sigma_clip algorithm until converged, and we've identified the outliers
filtered_data = sigma_clip(residuals, sig=args.sigma, iters=None)
mask = filtered_data.mask
# sigma0 = config['region_priors']['sigma0']
# logAmp = config["region_params"]["logAmp"]
# sigma = config["region_params"]["sigma"]
# Sort in decreasing strength of residual
nregions = 0
mus = []
for w, resid in sorted(zip(wl[mask], np.abs(residuals[mask])), key=itemgetter(1), reverse=True):
if w in wl[covered]:
continue
else:
# check to make sure region is not *right* at the edge of the echelle order
def compare_results(file_list, parameter_change_list, bary_starname,
orbital_parameters, objects, servaldir):
"""
Compares result files to find the best rv scatter around literature fit returns change in parameter that yielded best results
file_list: list of `str`
list containing the file paths of the files to be compared
parameter_change_list: list of `int`
list containing the parameter exponent shifts used to create the files in file_list
orbital_parameters : list of `float`
orbital_parameters = [K, P, e, omega, T0]
parameters of the keplerian fit to be used as "true" baseline
"""
sigma_list = np.zeros(len(file_list)) + 100 # 100 is a fudge factor
for f, fil in enumerate(file_list):
#assumes order of file_listand parameter_change_list are matched. (maybe extract from file name?)
wobble_res = h5py.File(fil, 'r')
w_dates = wobble_res['dates'][()]
w_dates_utc = wobble_res['dates_utc'][()]
w_RVs = wobble_res['star_time_rvs'][()]
w_RVs_original = w_RVs
w_RVs_er = wobble_res['star_time_sigmas'][()]
#barycorr for wobble_orig
from scipy.constants import codata
lightvel = codata.value('speed of light in vacuum') #for barycorr
# CAHA Coordinates for barycorr
_lat = 37.2236
_lon = -2.54625
_elevation = 2168.
w_RVs_original_barycorr = np.zeros(len(w_dates))
for n in tqdm(range(len(w_RVs_original_barycorr))):
w_RVs_original_barycorr[n] = bary.get_BC_vel(
w_dates_utc[n],
starname=bary_starname,
lat=_lat,
longi=_lon,
alt=_elevation,
zmeas=w_RVs_original[n] / lightvel)[0]
#Serval Correction
#read in SERVAL
ser_rvc = np.loadtxt(servaldir + objects[1] + "/" + objects[1] +
".rvc.dat")
# remove entries with nan in drift
ind_finitedrift = np.isfinite(ser_rvc[:, 3])
ser_rvc = ser_rvc[ind_finitedrift]
ser_corr = -ser_rvc[:, 8] - ser_rvc[:, 3]
#match wobble and serval
indices_serval = []
indices_wobble = []
for n in range(len(w_dates)):
ind_jd = np.where(
np.abs(ser_rvc[:, 0] - w_dates[n]) == np.nanmin(
np.abs(ser_rvc[:, 0] - w_dates[n])))[0][0]
if (ser_rvc[ind_jd, 0] - w_dates[n]
) * 24 * 60 < 20.: #only takes matches closer than 20 minutes
indices_serval.append(ind_jd)
indices_wobble.append(n)
print("#serval_ind:" + str(len(indices_serval)),
"#wobble_ind:" + str(len(indices_wobble)))
#now set up all the data according to the indices
ser_rvc = ser_rvc[indices_serval]
ser_corr = ser_corr[indices_serval]
w_dates = w_dates[indices_wobble]
w_dates_utc = w_dates_utc[indices_wobble]
w_RVs_original_barycorr = w_RVs_original_barycorr[
indices_wobble] + ser_corr
w_RVs_er = w_RVs_er[indices_wobble]
def fit_func(t, T0_offset):
return rv.radial_velocity(t, orbital_parameters[0],
orbital_parameters[1],
orbital_parameters[2],
orbital_parameters[3],
orbital_parameters[4] + T0_offset)
#fit to Wobble
xdata = w_dates
ydata = w_RVs_original_barycorr - np.nanmean(w_RVs_original_barycorr)
popt, pcov = sp.optimize.curve_fit(fit_func,
xdata,
ydata,
sigma=w_RVs_er,
absolute_sigma=True)
print("T0_offset Wobble = ", popt)
T0_offset = popt[0]
#make these weighted (maybe: thsi may not be a good idea if residuals are not strongly correlated to error (as with wobble results))
sigma_wob = np.nanstd(
sigma_clip(w_RVs_original_barycorr -
np.nanmean(w_RVs_original_barycorr) -
fit_func(w_dates, T0_offset),
sigma=5))
sigma_list[f] = sigma_wob
#TODO include some nice progress plots
best_index = np.argmin(sigma_list)
return parameter_change_list[best_index]
def mask_outliers(self, data, n_sigma=3, *args):
return sigma_clip(data, n_sigma).mask
def trace_mismatch(self, maxsep=None, sigma=None, minmax=None, synced=False):
"""
Return the mismatches between the mask and trace positions.
Based on the best-fitting (or fixed) offset and scale
parameters, :func:`match` is executed, forcing the
slit-mask and trace positions pairs to be uniquely matched.
The set of slit-mask positions without a matching trace are
identified by finding those slits in the range relevant to
the list of trace coordinates (see `minmax`), but without a
matching trace index.
.. todo::
explain synced adjustment
The set of mask-to-trace matches are identified as "bad" if
they meet any of the following criteria:
- The trace has not been masked (see :attr:`trace_mask`)
- A unique match could not be found (see :func:`match`)
- The absolute value of the separation is larger than the
provided `maxsep` (when `maxsep` is not None).
- The separation is rejected by a sigma-clipping (see
`sigma`)
Note that there is currently no argument that disables the
determination of bad traces. However, bad traces are simply
returned by the method; this function changes none of the
class attributes.
Args:
maxsep (:obj:`float`, optional):
The maximum allowed separation between the calibrated
coordinates of the designed slit position in pixels
and the matched trace. If None, use :attr:`maxsep`;
see :func:`find_best_match`.
sigma (:obj:`float`, optional):
The sigma value to use for rejection. If None, use
:attr:`sigma`; see :func:`find_best_match`.
minmax (array-like, optional):
A two-element array with the minimum and maximum
coordinate value to match to the trace data. If None,
this is determined from :attr:`trace_spat` and the
standard deviation of the fit residuals.
synced (:obj:`bool`, optional):
The mask coordinates being matched to are synced
left-to-right in adjacent pairs. I.e., the indices of
left edges are all even and the indices of all right
edges are odd.
Returns:
Two `numpy.ndarray`_ objects are returned: (1) the
indices of mask positions without a matching trace
position and (2) the list of trace positions identified
as "bad."
"""
# Check parameters are available
if self.par is None:
raise ValueError('No parameters are available.')
# Set the parameters using the relevant attributes if not provided directly
if maxsep is None:
maxsep = self.maxsep
if sigma is None:
sigma = self.sigma
# Get the coordinates and the matching indices: always use the
# existing parameters and force the matches to be unique.
_match_coo, _match_separation, _match_index = self.match(unique=True)
# Selection of positions included in the fit with valid match
# indices
gpm = numpy.invert(self.trace_mask) & (_match_index >= 0)
# Find the overlapping region between the trace and slit-mask
# coordinates
if minmax is None:
stddev = numpy.std(_match_separation[gpm])
_minmax = numpy.array([numpy.amin(self.trace_spat) - 3*stddev,
numpy.amax(self.trace_spat) + 3*stddev])
else:
_minmax = numpy.atleast_1d(minmax)
if _minmax.size != 2:
raise ValueError('`minmax` must be a two-element array.')
overlap = (_match_coo > _minmax[0]) & (_match_coo < _minmax[1])
# Find any large separations
bad = self.trace_mask | (_match_index < 0)
if maxsep is None:
diff = numpy.ma.MaskedArray(_match_separation, mask=bad)
kwargs = {} if sigma is None else {'sigma': sigma}
bad = numpy.ma.getmaskarray(sigma_clip(data=diff, **kwargs))
else:
bad[gpm] = numpy.absolute(_match_separation[gpm]) > maxsep
if synced:
# Get the union of all indices with good or missing matches
indx = numpy.array(list(set(numpy.append(numpy.where(overlap)[0],
_match_index[gpm & numpy.invert(bad)]))))
# If the coordinate are left-right synchronized, there
# should be an even number of indices, and the sorted
# sequence should always have adjacent pairs the are
# different by one
unsynced = numpy.where(numpy.diff(indx)[::2] != 1)[0]*2
if len(unsynced) != 0:
offset = numpy.ones(len(unsynced), dtype=int)
offset[indx[unsynced] % 2 == 1] = -1
overlap[indx[unsynced]+offset] = True
# Make sure the last index is paired
if indx[-1] % 2 == 0:
# Add a right
overlap[indx[-1]+1] = True
# Use these to construct the list of missing traces and those
# that are unmatched to the mask
return numpy.array(list(set(numpy.where(overlap)[0])
- set(_match_index[gpm & numpy.invert(bad)]))), \
numpy.where(bad & numpy.invert(self.trace_mask))[0]
def find_peaks(flux,
window=51,
niter=5,
clip_iter=5,
clip_sigma_upper=5.0,
clip_sigma_lower=5.0,
detection_sigma=3.0,
min_peak_dist_sigma=5.0,
gaussian_width=1.0,
make_fig=False):
"""
* Subtract median filter (param "window")
* Iterate: (param "niter")
* Sigma clip, estimate noise (params clip_iter, clip_sigma_upper clip_sigma_lower)
* Find peaks (param detection_sigma)
* Remove peaks too close to previous (param min_peak_dist_sigma)
* Fit Gaussians to peaks (initialize width at param gaussian_width)
Returns:
allpeakx: locations of peaks
fullmodel: the model of all the gaussians
If make_fig=True: fig, a plot showing all the peaks found at each iteration.
"""
# This is the data we will try to fit with a
# combination of Gaussians
xarr = np.arange(len(flux))
flux = flux - signal.medfilt(flux, window)
continuum = models.Linear1D(slope=0, intercept=0)
fullmodel = continuum
allpeakx = []
allpeaksigma = []
fitter = fitting.LevMarLSQFitter()
if make_fig: fig, axes = plt.subplots(niter)
for iiter in range(niter):
# Subtract existing peaks
tflux = flux - fullmodel(xarr)
# Estimate noise
cflux = sigma_clip(tflux,
iters=clip_iter,
sigma_upper=clip_sigma_upper,
sigma_lower=clip_sigma_lower)
noise = np.std(cflux)
# Find peaks in residual using gradient = 0
# Only keep peaks above detection threshold
deriv = np.gradient(tflux)
peaklocs = (deriv[:-1] >= 0) & (deriv[1:] < 0) & \
(tflux[:-1] > detection_sigma * noise)
peakx = np.where(peaklocs)[0]
peaky = flux[:-1][peaklocs]
# Prune peaks too close to existing peaks
peaks_to_keep = np.ones_like(peakx, dtype=bool)
for ix, x in enumerate(peakx):
z = (x - np.array(allpeakx)) / np.array(allpeaksigma)
if np.any(np.abs(z) < min_peak_dist_sigma):
peaks_to_keep[ix] = False
peakx = peakx[peaks_to_keep]
peaky = peaky[peaks_to_keep]
# Add new peaks to the model
for x, y in zip(peakx, peaky):
g = models.Gaussian1D(amplitude=y, mean=x, stddev=gaussian_width)
fullmodel = fullmodel + g
print("iter {}: {} peaks (found {}, added {})".format(
iiter,
fullmodel.n_submodels() - 1, len(peaks_to_keep), len(peakx)))
# Fit the full model
fullmodel = fitter(fullmodel,
xarr,
flux,
maxiter=200 * (fullmodel.parameters.size + 1))
print(fitter.fit_info["message"], fitter.fit_info["ierr"])
# Extract peak x and sigma
peak_x_indices = np.where(
["mean_" in param for param in fullmodel.param_names])[0]
peak_y_indices = peak_x_indices - 1
peak_sigma_indices = peak_x_indices + 1
allpeakx = fullmodel.parameters[peak_x_indices]
allpeaky = fullmodel.parameters[peak_y_indices]
allpeaksigma = fullmodel.parameters[peak_sigma_indices]
# Make a plot
if make_fig:
try:
ax = axes[iiter]
except:
ax = axes
ax.plot(xarr, flux)
ax.plot(peakx, peaky, 'ro')
ax.plot(xarr, fullmodel(xarr), lw=1)
ax.axhspan(-noise, +noise, color='k', alpha=.2)
ax.plot(xarr, flux - fullmodel(xarr))
ax.vlines(allpeakx,
allpeaky * 1.1,
allpeaky * 1.1 + 300,
color='r',
lw=1)
if make_fig: return allpeakx, fullmodel, fig
return allpeakx, fullmodel
def findLine(self):
img = fits.getdata(self.fullPath).astype(np.float32)
img[img < 0] = 0
img0 = fits.getdata(self.fullPath0).astype(np.float32)
img0[img0 < 0] = 0
imgDiff = img.copy() - img0.copy()
#imgDiff[imgDiff<0]=0
#new_hdu = fits.PrimaryHDU(imgDiff)
#new_hdu.writeto("f:/aa.fit")
'''
zimg = img.copy().astype(np.int)
zmin, zmax = zscale_image(zimg)
zimg[zimg>zmax] = zmax
zimg[zimg<zmin] = zmin
if zmin==zmax:
self.lines = np.array([])
return;
zimg=(((zimg-zmin)/(zmax-zmin))*255).astype(np.uint8)
'''
'''
Image.fromarray(zimg).save("f:/aa.png")
zimg = imgDiff.copy().astype(np.int)
zmin, zmax = zscale_image(zimg)
zimg[zimg>zmax] = zmax
zimg[zimg<zmin] = zmin
zimg=(((zimg-zmin)/(zmax-zmin))*255).astype(np.uint8)
Image.fromarray(zimg).save("f:/aa1.png")
return
'''
zimgDiff = imgDiff.copy()
zimgDiff[zimgDiff < 0] = 0
zimgDiff = cv2.medianBlur(zimgDiff, 3) #medianBlur blur
tmin = zimgDiff.min()
tmax = zimgDiff.max()
if tmin == tmax:
self.lines = np.array([])
return
zimgDiff = (((zimgDiff - tmin) / (tmax - tmin)) * 255.0).astype(
np.uint8)
#Image.fromarray(zimgDiff).save(r"E:\work\program\python\OTSimulation\meteor-find\aa1.png")
self.img = img
self.zimg = img
self.imgDiff = imgDiff
self.zimgDiff = zimgDiff
self.imgH = img.shape[0]
self.imgW = img.shape[1]
#drop image border 20pix
#zimgDiff[0:self.borderWidth, :] = 0
#zimgDiff[:, 0:self.borderWidth] = 0
#zimgDiff[self.imgH-self.borderWidth:self.imgH, :] = 0
#zimgDiff[:, self.imgW-self.borderWidth:self.imgW] = 0
imgAvg = np.mean(zimgDiff)
imgRms = np.std(zimgDiff)
thred1 = imgAvg + 3 * imgRms
self.imgAvg1 = imgAvg
self.imgRms1 = imgRms
self.thred1 = thred1
print("avg1=%f;rms1=%f;thred1=%f" % (imgAvg, imgRms, thred1))
#new_hdu = fits.PrimaryHDU(zimgDiff)
#new_hdu.writeto("f:/ab.fit")
timg1 = zimgDiff[zimgDiff > thred1]
if len(timg1) == 0:
self.lines = np.array([])
return
filtered_data = sigma_clip(timg1, sigma=3, iters=2, copy=False)
imgbg = filtered_data.data[~filtered_data.mask]
imgAvg = np.average(imgbg)
imgRms = np.std(imgbg)
#thred2 = imgAvg + 3 * imgRms
thred2 = imgAvg + 2 * imgRms
self.imgAvg2 = imgAvg
self.imgRms2 = imgRms
self.thred2 = thred2
print("avg1=%f;rms1=%f;thred2=%f" % (imgAvg, imgRms, thred2))
thred2 = (145 if thred2 > 145 else thred2)
#flag,bimg = cv2.threshold(img,0,255,cv2.THRESH_OTSU)
flag, bimg = cv2.threshold(zimgDiff, thred2, 255, cv2.THRESH_BINARY)
#Image.fromarray(bimg).save("f:/aa1.png")
rho = 1
theta = np.pi / 180
#lines = cv2.HoughLinesP(bimg, rho = rho, theta = theta, threshold = 10,
# minLineLength = 50,maxLineGap = self.maxLineGap)
#寻找暗的轨迹
#lines = cv2.HoughLinesP(bimg, rho = rho, theta = theta, threshold = 20,
# minLineLength = 50,maxLineGap = 20)
#寻找亮的轨迹，亮的流星
lines = cv2.HoughLinesP(bimg,
rho=rho,
theta=theta,
threshold=50,
minLineLength=50,
maxLineGap=self.maxLineGap)
while lines is None and thred2 > 100:
#flag,bimg = cv2.threshold(img,0,255,cv2.THRESH_OTSU)
flag, bimg = cv2.threshold(zimgDiff, thred2, 255,
cv2.THRESH_BINARY)
lines = cv2.HoughLinesP(bimg,
rho=rho,
theta=theta,
threshold=50,
minLineLength=50,
maxLineGap=self.maxLineGap)
if lines is None:
thred2 = thred2 - 5
lines = (np.array([]) if lines is None else lines)
print("thred1: %f thred2: %f lines: %d" %
(thred1, thred2, lines.shape[0]))
self.bimg = bimg
self.lines = lines
def rescale_snr(specwave,
flux=None,
ivar=None,
x1=None,
x2=None,
cont=None,
cont_kernel=51,
cont_Niter=3,
make_fig=False,
**sigma_clip_kwargs):
"""
Take biweight standard deviation of x_i/sigma_i of sigma-clipped data,
rescale ivar so that standard deviation is 1
Returns a Spectrum1D object
"""
if flux is None and ivar is None:
assert isinstance(specwave, Spectrum1D)
spec = specwave
wave, flux, ivar = spec.dispersion, spec.flux, spec.ivar
meta = spec.metadata
else:
wave = specwave
assert len(wave) == len(flux)
assert len(wave) == len(ivar)
meta = OrderedDict({})
if cont is None:
cont = fast_find_continuum(flux, cont_kernel, cont_Niter)
else:
assert len(cont) == len(flux)
errs = ivar**-0.5
errs[errs > 10 * flux] = np.nan
iirescale = np.ones_like(wave, dtype=bool)
if x1 is not None: iirescale = iirescale & (wave > x1)
if x2 is not None: iirescale = iirescale & (wave < x2)
norm = flux[iirescale] / cont[iirescale]
normerrs = errs / cont[iirescale]
z = (norm - 1.) / normerrs
clipped = sigma_clip(z[np.isfinite(z)], **sigma_clip_kwargs)
noise = biweight_scale(clipped[~clipped.mask])
print("Noise is {:.2f} compared to 1.0".format(noise))
new_ivar = ivar / (noise**2.)
outspec = Spectrum1D(wave, flux, new_ivar, meta)
if make_fig:
newz = z / noise
newerrs = errs * noise
newnormerrs = normerrs * noise
import matplotlib.pyplot as plt
fig, axes = plt.subplots(2, 3, figsize=(12, 6))
ax = axes[0, 0]
ax.plot(wave, flux)
ax.plot(wave, errs)
ax.plot(wave, newerrs)
ax.plot(wave, cont, color='k', ls=':')
ax.set_xlabel('wavelength')
ax.set_ylabel('counts')
ax = axes[1, 0]
ax.plot(wave, norm)
ax.plot(wave, normerrs)
ax.plot(wave, newnormerrs)
ax.axhline(1, color='k', ls=':')
ax.set_xlabel('wavelength')
ax.set_ylabel('norm')
ax.set_ylim(0, 1.2)
ax = axes[1, 1]
ax.plot(wave, z)
ax.plot([np.nan], [np.nan]) # hack to get the right color
ax.plot(wave, newz)
ax.axhline(0, color='k', ls=':')
ax.set_xlabel('wavelength')
ax.set_ylabel('z')
ax.set_ylim(-7, 7)
ax = axes[0, 1]
bins = np.linspace(-7, 7, 100)
binsize = np.diff(bins)[1]
ax.plot(bins,
norm_distr.pdf(bins) * np.sum(np.isfinite(z)) * binsize,
color='k')
ax.hist(z[np.isfinite(z)], bins=bins)
ax.hist(clipped[~clipped.mask], bins=bins, histtype='step')
ax.hist(newz[np.isfinite(newz)], bins=bins, histtype='step')
ax.set_xlabel('z')
ax.set_xlim(-7, 7)
ax = axes[0, 2]
zfinite = z.copy()
zfinite[~np.isfinite(zfinite)] = 0.
autocorr = np.correlate(zfinite, zfinite, mode="same")
ax.plot(np.arange(len(flux)), autocorr, '.-')
ax.axvline(len(flux) // 2)
ax.set_xlim(
len(flux) // 2 - 10,
len(flux) // 2 + 10,
)
ax.set_xlabel("pixel")
ax.set_ylabel("autocorrelation(z)")
z1, z2 = -10, 10
zarr1 = np.zeros((len(z) - 1, 2))
zarr1[:, 0] = z[:-1]
zarr1[:, 1] = z[1:]
zarr1 = zarr1[np.sum(np.isfinite(zarr1), axis=1) == 2]
zarr2 = np.zeros((len(z) - 2, 2))
zarr2[:, 0] = z[:-2]
zarr2[:, 1] = z[2:]
zarr2 = zarr2[np.sum(np.isfinite(zarr2), axis=1) == 2]
#ax = axes[0,2]
#ax.plot([z1,z2],[z1,z2],'k:')
#ax.plot(z[:-1], z[1:], '.', alpha=.3)
#ax.set_title("r={:+.2}".format(pearsonr(zarr1[:,0],zarr1[:,1])[0]))
#ax.set_xlabel("z(pixel)"); ax.set_ylabel("z(pixel+1)")
#ax.set_xlim(z1,z2); ax.set_ylim(z1,z2)
ax = axes[1, 2]
ax.plot([z1, z2], [z1, z2], 'k:')
ax.plot(z[:-2], z[2:], '.', alpha=.3)
ax.set_title("r={:+.2}".format(pearsonr(zarr2[:, 0], zarr2[:, 1])[0]))
ax.set_xlabel("z(pixel)")
ax.set_ylabel("z(pixel+2)")
ax.set_xlim(z1, z2)
ax.set_ylim(z1, z2)
fig.tight_layout()
return fig, outspec, noise
return outspec, noise
print('Total of ', numtotint, ' integrations for this filter')
if numtotint < 10:
numsigma = 5
meddata3d = np.median(normdata3d, axis=2, keepdims=True)
datadiffs = normdata3d - meddata3d
datadiffsdiverr = datadiffs / normerr3d
datadiffsdiverrmasked = np.ma.masked_greater_equal(
datadiffsdiverr, numsigma)
clippeddata3d = np.ma.masked_array(normdata3d,
datadiffsdiverrmasked.mask)
clippederr3d = np.ma.masked_array(normerr3d, clippeddata3d.mask)
else:
numsigma = 3.0
clippeddata3d = sigma_clip(normdata3d,
sigma=numsigma,
maxiters=2,
axis=2,
cenfunc='median')
clippederr3d = np.ma.masked_array(normerr3d, clippeddata3d.mask)
#For data array use mean of clipped array
meanclippeddata = np.ma.mean(clippeddata3d, axis=2)
#For error array add errors in quadrature and divide by number of unmasked samples
meanclippederr = (1.0 / (np.sum(
(~clippederr3d.mask), axis=2))) * (np.ma.sum(
(clippederr3d**2), axis=2))**0.5
#Patch unusual blob regions in some filter images, replacing with the F200W flat that doesn't show them.
if filterdirlist[l] != 'F200W':
hdulistf200w = fits.open('jwst_niriss_cv3_imageflat_F200W.fits')
dataf200w = hdulistf200w['SCI'].data
#These are x,y positions for photutils
def master_ff(flattype, masterb, masterf):
""" Produces a master flat that has been bias corrected """
flat_fn = []
for file in glob.glob('*.fits'):
hdulist = fits.open(file)
header = hdulist[0].header
if flattype == 'IMSKY':
if (header['object'] == 'SKY,FLAT'
and header['HIERARCH ESO DPR TECH'] == 'IMAGE'):
flat_fn.append(file)
if flattype == 'POLDOME':
if (header['object'] == 'DOME'
and header['HIERARCH ESO DPR TECH'] == 'POLARIMETRY'):
flat_fn.append(file)
hdulist.close()
print("Number of flat frames in directory:", len(flat_fn))
fstack_data = np.zeros([len(flat_fn), 1024, 1024])
mblist = fits.open(masterb)
mbdata = mblist[0].data
mblist.close()
i = 0
for file in flat_fn:
hdulist = fits.open(file)
data = hdulist[0].data
header = hdulist[0].header
if flattype == 'IMSKY':
if (np.shape(data) == (1030, 1030) and header['CDELT1'] == 2
and header['CDELT1'] == 2):
bc_data = data[:1024, :1024] - mbdata
clipped_data = sigma_clip(bc_data,
sigma_upper=3,
sigma_lower=104)
bc_data[clipped_data.mask == True] = np.nan
fstack_data[i, :, :] = bc_data
i += 1
if flattype == 'POLDOME':
if (np.shape(data) == (1030, 1030) and header['CDELT1'] == 2
and header['CDELT1'] == 2):
bc_data = data[:1024, :1024] - mbdata
fstack_data[i, :, :] = bc_data
i += 1
hdulist.close()
print("Number of flat frames used in master flat:", i)
if i > 0:
fstack_sum = np.nanmean(fstack_data, axis=0)
fmedian = np.nanmedian(fstack_sum)
fstack_sum[np.isnan(fstack_sum)] = fmedian
fstack_sum[fstack_sum <= 0] = fmedian
flat_array = fstack_sum / fmedian
fstd = np.nanstd(flat_array)
hdu = fits.PrimaryHDU(flat_array)
if flattype == 'IMSKY':
hdu.header['object'] = 'MASTER SKY,FLAT'
hdu.header['obstech'] = 'IMAGE'
if flattype == 'POLDOME':
hdu.header['object'] = 'MASTER DOME,FLAT'
hdu.header['obstech'] = 'POLARIMETRY'
hdu.header['frames'] = len(flat_fn)
hdu.header['median'] = 1
hdu.header['std'] = fstd
hdu.header['naxis1'] = 1024
hdu.header['naxis2'] = 1024
hdu.header['binning'] = 2
hdulist = fits.HDUList([hdu])
hdulist.writeto(masterf)
return 0
else:
raise FileNotFoundError("No applicable flat frames")
def A_photometry(image_data,
bg_err,
factor=1,
ape_sum=[],
ape_sum_err=[],
cx=15,
cy=15,
r=2.5,
a=5,
b=5,
w_r=5,
h_r=5,
theta=0,
shape='Circular',
method='center'):
'''
Performs aperture photometry, first by creating the aperture (Circular,
Rectangular or Elliptical), then it sums up the flux that falls into the
aperture.
Parameters
==========
image_data: 3D array
Data cube of images (2D arrays of pixel values).
bg_err : 1D array
Array of uncertainties on pixel value.
factor : float (optional)
Electron count to photon count factor. Default is 1 if none given.
ape_sum : 1D array (optional)
Array of flux to append new flux values to. If 'None', the new values
will be appended to an empty array
ape_sum_err: 1D array (optional)
Array of flux uncertainty to append new flux uncertainty values to. If
'None', the new values will be appended to an empty array.
cx : float or 1D array (optional)
x-coordinate of the center of the aperture. Dimension must be equal to
dimension of cy. Default is 15.
cy : float or 1D array (optional)
y-coordinate of the center of the aperture. Default is 15.
r : int (optional)
If phot_meth is 'Aperture' and ap_shape is 'Circular', c_radius is
the radius for the circular aperture. Default is 2.5.
a : int (optional)
If phot_meth is 'Aperture' and ap_shape is 'Elliptical', e_semix is
the semi-major axis for elliptical aperture (x-axis). Default is 5.
b : int (optional)
If phot_meth is 'Aperture' and ap_shape is 'Elliptical', e_semiy is
the semi-major axis for elliptical aperture (y-axis). Default is 5.
w_r : int (optional)
If phot_meth is 'Aperture' and ap_shape is 'Rectangular', r_widthx is
the full width for rectangular aperture (x-axis). Default is 5.
h_r : int (optional)
If phot_meth is 'Aperture' and ap_shape is 'Rectangular', r_widthy is
the full height for rectangular aperture (y-axis). Default is 5.
theta : int (optional)
If phot_meth is 'Aperture' and ap_shape is 'Elliptical' or
'Rectangular', theta is the angle of the rotation angle in radians
of the semimajor axis from the positive x axis. The rotation angle
increases counterclockwise. Default is 0.
shape : string object (optional)
If phot_meth is 'Aperture', ap_shape is the shape of the aperture.
Possible aperture shapes are 'Circular', 'Elliptical', 'Rectangular'.
Default is 'Circular'.
method : string object (optional)
If phot_meth is 'Aperture', apemethod is the method used to
determine the overlap of the aperture on the pixel grid. Possible
methods are 'exact', 'subpixel', 'center'. Default is 'exact'.
Returns
-------
ape_sum : 1D array
Array of flux with new flux appended.
ape_sum_err: 1D array
Array of flux uncertainties with new flux uncertainties appended.
'''
l, h, w = image_data.shape
# central position of aperture
#position = np.c_[cx, cy] # remove when uncommenting below
if (type(cx) is list):
position = np.c_[cx, cy]
else:
position = np.c_[(cx * np.ones(l)), (cy * np.ones(l))]
tmp_sum = []
tmp_err = []
# performing aperture photometry
for i in range(l):
#aperture = CircularAperture(position[i], r=r) # remove when uncommenting below
if (shape == 'Circular'):
aperture = CircularAperture(position[i], r=r)
elif (shape == 'Elliptical'):
aperture = EllipticalAperture(position[i], a=a, b=b, theta=theta)
elif (shape == 'Rectangular'):
aperture = RectangularAperture(position[i],
w=w_r,
h=h_r,
theta=theta)
data_error = calc_total_error(image_data[i, :, :],
bg_err[i],
effective_gain=1)
phot_table = aperture_photometry(image_data[i, :, :],
aperture,
error=data_error,
pixelwise_error=False,
method=method)
tmp_sum.extend(phot_table['aperture_sum'] * factor)
tmp_err.extend(phot_table['aperture_sum_err'] * factor)
# removing outliers
tmp_sum = sigma_clip(tmp_sum, sigma=4, iters=2, cenfunc=np.ma.median)
tmp_err = sigma_clip(tmp_err, sigma=4, iters=2, cenfunc=np.ma.median)
ape_sum.extend(tmp_sum)
ape_sum_err.extend(tmp_err)
return ape_sum, ape_sum_err
