def make_plot(days_ago, dates, mag):
print('Making plot...')
time_span = np.max(dates) - np.min(dates)
flatten_lc, trend_lc = flatten(
days_ago,
mag,
method='lowess',
window_length=time_span/3,
return_trend=True,
)
plt.scatter(days_ago, mag, s=5, color='blue', alpha=0.5)
plt.scatter(days_ago1, all_mags1, s=10, color='black', alpha=0.8, marker="x")
plt.xlabel('Days before today')
plt.ylabel('Visual magnitude')
mid = biweight_location(mag)
plt.ylim(mid-1, mid+1)
plt.xlim(-1, 20)
plt.plot(days_ago, trend_lc, color='red', linewidth=1)
plt.gca().invert_yaxis()
plt.gca().invert_xaxis()
date_text = datetime.datetime.now().strftime("%d %b %Y")
data_last24hrs = np.where(days_ago<1)
mean_last24hrs = biweight_location(mag[data_last24hrs])
lumi = str(format(mean_last24hrs, '.2f'))
plt.text(19.5, mid+1-0.25, "AAVSO observations visual (by-eye) in blue", color='blue')
plt.text(19.5, mid+1-0.15, "AAVSO observations from CCDs in black", color='black')
plt.text(19.5, mid+1-0.05, "LOESS trend in red", color='red')
plt.text(19.5, mid-1+0.1, '#Betelgeuse brightness ' + lumi + " mag on " + date_text + " by @betelbot")
plt.savefig(plot_file, bbox_inches='tight', dpi=300)
print('Done.')
def plot_2D_offsets(df, stat_key, prefices=["Delta_x_", "Delta_y_"],
suffices=["_x", "_y"], prefix_bool=1):
for i, key in enumerate(stat_key) :
plt.axes().set_aspect('equal')
if prefix_bool:
fixed_keys = [prefices[0] + key, prefices[1] + key]
else:
fixed_keys = [key + suffices[0], key + suffices[1]]
plt.plot(
np.array(df[fixed_keys[0]]),
np.array(df[fixed_keys[1]]), 'b.', alpha=0.05
)
biweight_loc = (
biweight_location(df[fixed_keys[0]]),
biweight_location(df[fixed_keys[1]]))
# The red cross is the biweight location along each dimension
plt.plot(biweight_loc[0], biweight_loc[1],
'rx', mew=2.)
plt.tick_params(labeltop='off', labelright='off')
plt.axes().yaxis.set_ticks_position('left')
plt.axes().xaxis.set_ticks_position('bottom')
plt.xlim(-300, 300)
plt.ylim(-300, 300)
plt.title(key + ', biweight_loc = {0:.2f}, {1:.2f}'.format(
*biweight_loc))
plt.show()
plt.clf()
def plot_hist_errors(pasta, y_Z, y_age, y_Zmin=-3, y_Zmax=3, y_agemin=-3, y_agemax=3,
label_Z='label_Z', label_age='label_age', file='test'):
location = [biweight_location(y_Z), biweight_location(y_age)]
scale = [biweight_scale(y_Z), biweight_scale(y_age)]
fig = plt.figure(constrained_layout=False)
gs = fig.add_gridspec(nrows=1, ncols=2) # , width_ratios=[2, 1], height_ratios=[1, 2])
a = fig.add_subplot(gs[0, 0])
n, bins_Z, patches = plt.hist(y_Z, orientation='vertical', color='khaki')
plt.vlines(location[0] + scale[0], 0, np.max(n), label='scale=' + str(scale[0]))
plt.vlines(location[0] - scale[0], 0, np.max(n))
plt.vlines(location[0], 0, np.max(n), color='blue', label='location=' + str(location[0]))
plt.xlabel(label_Z)
plt.xlim(y_Zmin, y_Zmax)
plt.legend()
a = fig.add_subplot(gs[0, 1])
n, bins_age, patches = plt.hist(y_age, orientation='vertical', color='khaki')
plt.vlines(location[1] + scale[1], 0, np.max(n), label='scale=' + str(scale[1]))
plt.vlines(location[1] - scale[1], 0, np.max(n))
plt.vlines(location[1], 0, np.max(n), color='blue', label='location=' + str(location[1]))
plt.legend()
plt.xlabel(label_age)
plt.xlim(y_agemin, y_agemax)
'''
'''
# plt.legend()
plt.tight_layout()
plt.savefig('results/' + pasta + '/' + file + '.png', dpi=300, bbox_inches='tight')
plt.close()
def getbiweight(z):
biweightscale=biweight_midvariance(z)
biweightlocation=biweight_location(z)
flag=abs(z-biweightlocation)/biweightscale < scale_cut
#flag=np.ones(len(z),'bool')
repeatflag=1
nloop=0
#print biweightlocation, biweightscale
oldbiweightscale=biweightscale
while repeatflag:
newdata=z[flag]
biweightscale=biweight_midvariance(newdata, M=biweightlocation)
biweightlocation=biweight_location(newdata, M=biweightlocation)
oldflag = flag
#flag=abs(z-biweightlocation)/biweightscale < scale_cut
nloop += 1
#print nloop, biweightlocation, biweightscale, len(newdata), sum(flag)
#if sum(flag == oldflag) == len(flag):
# repeatflag=0
# print 'nloop = ', nloop
if abs(biweightscale - oldbiweightscale) < 1.:
repeatflag=0
#print 'nloop = ', nloop
if nloop > 5:
repeatflag = 0
oldbiweightscale=biweightscale
#print nloop, biweightlocation, biweightscale
#flag=abs(z-biweightlocation)/biweightscale < 4.
#biweightscale=biweight_midvariance(z[flag],M=biweightlocation)
return biweightlocation, biweightscale
def find_matches(sources, xc, yc):
''' Matching sources using closest neighbor and clipping '''
dx = (sources['xcentroid'][:, np.newaxis] - xc)
dy = (sources['ycentroid'][:, np.newaxis] - yc)
dd = np.sqrt(dx**2 + dy**2)
ind = np.argmin(dd, axis=1)
dxk = dx[np.arange(dx.shape[0]), ind]
dyk = dy[np.arange(dy.shape[0]), ind]
bvx = biweight_midvariance(dxk)
bvy = biweight_midvariance(dyk)
mvx = biweight_location(dxk)
mvy = biweight_location(dyk)
sel = np.where(
(np.abs(dxk - mvx) < 3. * bvx) * (np.abs(dyk - mvy) < 3. * bvy))[0]
return sel, ind[sel]
def plot_linearity(scifile, ivmfile):
"""
Assumes sky-subtracted images in counts/sec and corresponding IVM
:param scifile: science image file (cts/sec)
:param ivmfile: inverse variance map
"""
scih = fits.open(scifile)
ivmh = fits.open(ivmfile)
exptime = scih[0].header['EXPTIME']
scidata = scih[0].data
vardata = 1/ivmh[0].data
skyvar = biweight_location(vardata)
snr_cps = scidata / np.sqrt(vardata - skyvar)
# Only plot pixels that are at least 2 sigma above sky
mask = scidata > 2*np.sqrt(skyvar)
pp.scatter(scidata[mask].flat, snr_cps[mask].flat,
label='Predicted (Data*sqrt(Weight))')
# Idealized (poisson) SNR model is SNR(cts) = sqrt(cts)
x_ideal = np.linspace(0, scidata[mask].max(), 200)
y_ideal = np.sqrt(x_ideal*exptime)
pp.plot(x_ideal, y_ideal,
label='Ideal (sqrt(Counts))')
pp.xlabel('Signal (counts/sec)')
pp.ylabel('SNR (sky variance subtracted)')
pp.axis([x_ideal.min(), x_ideal.max(), y_ideal.min(), y_ideal.max()])
pp.legend(loc='lower right')
pp.show()
scih.close()
ivmh.close()
return
def calculate_statistics_no_clipping(self):
"""
This function ...
:return:
"""
# Compress (remove masked values)
flattened = np.ma.array(self.sources_with_noise.data,
mask=self.rotation_mask.data).compressed()
median = np.median(flattened)
biweight_loc = biweight_location(flattened)
biweight_midvar = biweight_midvariance(flattened)
median_absolute_deviation = mad_std(flattened)
#print("median", median)
#print("biweigth_loc", biweight_loc)
#print("biweight_midvar", biweight_midvar)
#print("median_absolute_deviation", median_absolute_deviation)
self.statistics.no_clipping = Map()
self.statistics.no_clipping.median = median
self.statistics.no_clipping.biweight_loc = biweight_loc
self.statistics.no_clipping.biweight_midvar = biweight_midvar
self.statistics.no_clipping.median_absolute_deviation = median_absolute_deviation
def compute_apcor(apcor_all, apcorlim):
"""
Filter out edge and other bad regions by selecting
the apcorlim greatest aperture correction values.
Then compute the biweight value and return the
resultant average aperture correction.
Parameters
----------
apcor_all : numpy:ndarray
the aperture corrections
apcorlim : int
the number of largest
values to consider
"""
flattened = sorted(apcor_all.flatten())
top_vals = np.flip(flattened, 0)[:apcorlim]
# Check if all elements are identical
# as this breaks biweight
if any(((top_vals - top_vals[0]) / top_vals[0]) > 1e-10):
return biweight_location(top_vals)
else:
_logger.warning("All aperture correction measurements the same!")
return top_vals[0]
def Multi_Band_GP(x_range, x, y, y_err, dim, n_samples=False, sampling=False):
""" Considers cross corrolations between multiple bands as dims, prone to holding the order of the bands too rigidly """
""" Will optimize for 'best' parameters when given no parameters """
""" x = mjd, y and y_err = measurment, dim and dim_err = wavelength in nm """
length_scale = 20
signal_to_noises = (np.abs(y) /
np.sqrt(np.power(y_err, 2) + (1e-2 * np.max(y))**2))
scale = np.abs(y[signal_to_noises.argmax()])
kernel = (
(0.5 * scale)**2 *
george.kernels.Matern32Kernel([length_scale**2, 6000**2], ndim=2))
kernel.freeze_parameter('k2:metric:log_M_1_1')
kernel.freeze_parameter('k1:log_constant') #Fixed Scale
x_data = np.vstack([x, dim]).T
gp = george.GP(kernel, mean=biweight_location(y))
guess_parameters = gp.get_parameter_vector()
gp.compute(x_data, y_err)
x_pred = np.linspace(x.min(), x.max(), n_samples)
x_pred = np.vstack([x, dim]).T
pred, pred_var = gp.predict(y, x_pred, return_var=True)
# bounds = [(0, np.log(1000**2))]
def neg_ln_like(p):
gp.set_parameter_vector(p)
return -gp.log_likelihood(y)
def grad_neg_ln_like(p):
gp.set_parameter_vector(p)
return -gp.grad_log_likelihood(y)
result = minimize(
neg_ln_like,
gp.get_parameter_vector(),
jac=grad_neg_ln_like,
# bounds=bounds
)
if result.success:
gp.set_parameter_vector(result.x)
else:
gp.set_parameter_vector(guess_parameters)
gp.set_parameter_vector(result.x)
# print(kernel.get_parameter_vector(True))
#print("\nFinal ln-likelihood: {0:.2f}".format(gp.log_likelihood(y)))
if n_samples != False:
x_pred = np.vstack([
np.array(
list(np.linspace(x_range.min(), x_range.max(), n_samples)) *
np.unique(dim).size),
np.array(np.sort(list(np.unique(dim)) * n_samples))
]).T
# x_pred = np.vstack([np.array(list(np.linspace(x_range.min(), x_range.max(), n_samples))*6),
# np.array(np.sort([357, 476, 621, 754, 870, 1004]*n_samples))]).T
pred, pred_var = gp.predict(y, x_pred, return_var=True)
output = [x_pred[:, 0], pred, np.sqrt(pred_var), x_pred[:, 1], []]
return output
elif sampling != False:
x_pred = np.vstack([np.array(sampling[0]), np.array(sampling[1])]).T
pred, pred_var = gp.predict(y, x_pred, return_var=True)
output = [x_pred[:, 0], pred, np.sqrt(pred_var), x_pred[:, 1], []]
return output
def calculate_reference_statistics(times, mags):
"""Calculate reference statistics for a light curve
We do this using biweight estimators for both the location and scale. One challenge
with this is that if there are many repeated observations, which is the case for
some of the ZTF deep fields, they can overwhelm all of the other observations. To
mitigate this, we only use the first observation from each night when calculating
the statistics.
Parameters
----------
times : ndarray
Time of each observation.
mags : ndarray
Magnitude of each observation.
Returns
-------
reference_magnitude : float
Reference magnitude
reference_scale : float
Reference scale
"""
# Consider at most one observation from each night.
_, indices = np.unique(times.astype(int), return_index=True)
use_mags = mags[indices]
# Use a robust estimator of the reference time and standard deviation.
reference_magnitude = biweight_location(use_mags)
reference_scale = biweight_scale(use_mags)
return reference_magnitude, reference_scale
def calculate_statistics_no_clipping(self):
"""
This function ...
:return:
"""
# Compress (remove masked values)
flattened = np.ma.array(self.sources_with_noise.data, mask=self.rotation_mask.data).compressed()
median = np.median(flattened)
biweight_loc = biweight_location(flattened)
biweight_midvar = biweight_midvariance(flattened)
median_absolute_deviation = mad_std(flattened)
#print("median", median)
#print("biweigth_loc", biweight_loc)
#print("biweight_midvar", biweight_midvar)
#print("median_absolute_deviation", median_absolute_deviation)
self.statistics.no_clipping = Map()
self.statistics.no_clipping.median = median
self.statistics.no_clipping.biweight_loc = biweight_loc
self.statistics.no_clipping.biweight_midvar = biweight_midvar
self.statistics.no_clipping.median_absolute_deviation = median_absolute_deviation
def base_reduction(filename, get_header=False):
# Load fits file
a = fits.open(filename)
image = np.array(a[0].data, dtype=float)
# Overscan subtraction
overscan_length = 32 * (image.shape[1] / 1064)
O = biweight_location(image[:, -(overscan_length - 2):])
image[:] = image - O
# Trim image
image = image[:, :-overscan_length]
# Gain multiplication (catch negative cases)
gain = a[0].header['GAIN']
gain = np.where(gain > 0., gain, 0.85)
rdnoise = a[0].header['RDNOISE']
rdnoise = np.where(rdnoise > 0., rdnoise, 3.)
amp = (a[0].header['CCDPOS'].replace(' ', '') +
a[0].header['CCDHALF'].replace(' ', ''))
try:
ampname = a[0].header['AMPNAME']
except:
ampname = None
header = a[0].header
# Orient image
a = orient_image(image, amp, ampname) * gain
# Calculate error frame
E = np.sqrt(rdnoise**2 + np.where(a > 0., a, 0.))
if get_header:
return a, E, header
return a, E
def build_XA(IFU, ww, skys, wstart, wend, amps):
# Select referece source
# here we will use as A the beiweight location (~ mean) from the entire IFU
XA = []
for k in skys[(IFU, amps[0])]:
amps_data = []
for amp in amps:
if k in skys[(IFU, amp)]:
amps_data.append(skys[(IFU, amp)][k])
stack = np.vstack(amps_data)
bloc = biweight_location(stack, axis=0)
XA.append(bloc)
XA = np.array(XA)
# hack to homogenize lengths, the rebinning does make sure
# that the wavelength grid always stars at the same wavelength
# but not necessarey, end at the same ( ther may be a few pixel more or less)
#N = np.min([XA.shape[1], XB.shape[2], ww.shape[0]])
N = np.min([XA.shape[1], ww.shape[0]])
ww = ww[:N]
XA = XA[:, :N]
# can't have nans
XA[np.isnan(XA)] = 0.
ii = (ww >= wstart) * (ww <= wend)
wwcut = ww[ii]
XAcut = XA[:, ii]
return wwcut, XAcut
def subtract_background(self):
"""Subtract the background levels from each band.
The background levels are estimated using a biweight location
estimator. This estimator will calculate a robust estimate of the
background level for objects that have short-lived light curves, and it
will return something like the median flux level for periodic or
continuous light curves.
Returns
-------
subtracted_observations : pandas.DataFrame
A modified version of the observations DataFrame with the
background level removed.
"""
subtracted_observations = self.observations.copy()
for band in self.bands:
mask = self.observations["band"] == band
band_data = self.observations[mask]
# Use a biweight location to estimate the background
ref_flux = biweight_location(band_data["flux"])
subtracted_observations.loc[mask, "flux"] -= ref_flux
return subtracted_observations
def calc_stat(data, sigma=1.8, niter=10, algorithm='median'):
"""Calculate statistics for given data.
Parameters
----------
data : ndarray
Data to be calculated from.
sigma : float
Sigma for sigma clipping.
niter : int
Number of iterations for sigma clipping.
algorithm : {'mean', 'median', 'mode', 'stddev'}
Algorithm for statistics calculation.
Returns
-------
val : float
Statistics value.
Raises
------
ValueError
Invalid algorithm.
"""
arr = np.ravel(data)
if len(arr) < 1:
return 0.0
kwargs = {'sigma': sigma}
if ASTROPY_LT_3_1:
kwargs['iters'] = niter
else:
kwargs['maxiters'] = niter
arr_masked = sigma_clip(arr, **kwargs)
arr = arr_masked.data[~arr_masked.mask]
if len(arr) < 1:
return 0.0
algorithm = algorithm.lower()
if algorithm == 'mean':
val = arr.mean()
elif algorithm == 'median':
val = np.median(arr)
elif algorithm == 'mode':
val = biweight_location(arr)
elif algorithm == 'stddev':
val = arr.std()
else:
raise ValueError('{0} is not a valid algorithm for sky background '
'calculations'.format(algorithm))
return val
def calc_stat(data, sigma=1.8, niter=10, algorithm='median'):
"""Calculate statistics for given data.
Parameters
----------
data : ndarray
Data to be calculated from.
sigma : float
Sigma for sigma clipping.
niter : int
Number of iterations for sigma clipping.
algorithm : {'mean', 'median', 'mode', 'stddev'}
Algorithm for statistics calculation.
Returns
-------
val : float
Statistics value.
Raises
------
ValueError
Invalid algorithm.
"""
arr = np.ravel(data)
if len(arr) < 1:
return 0.0
if ((astropy_version.major==1 and astropy_version.minor==0) or
(astropy_version.major < 1)):
arr_masked = sigma_clip(arr, sig=sigma, iters=niter)
else:
arr_masked = sigma_clip(arr, sigma=sigma, iters=niter)
arr = arr_masked.data[~arr_masked.mask]
if len(arr) < 1:
return 0.0
algorithm = algorithm.lower()
if algorithm == 'mean':
val = arr.mean()
elif algorithm == 'median':
val = np.median(arr)
elif algorithm == 'mode':
val = biweight_location(arr)
elif algorithm == 'stddev':
val = arr.std()
else:
raise ValueError('{0} is not a valid algorithm for sky background '
'calculations'.format(algorithm))
return val
def setup_fluxlim(args, rargs):
"""
This is equivalent to the rflim0 and rsetfl scripts.
Determine the input values for the flux limit calculation,
create the input file, create the slurm file using the jobsplitter
and launch it using sbatch
"""
nightshot = args.night + 'v' + args.shotid
dithall = DithAllFile(args.dithall_dir + '/' + nightshot + '/dithall.use')
if args.ifu_slots:
ifus = args.ifu_slots
else:
ifus = np.unique(dithall.ifuslot)
fname = 'flim%s' % nightshot
with open(fname, 'w') as f:
for ifu in ifus:
ifu_dith = dithall.where(dithall.ifuslot == ifu)
dist = np.sqrt(ifu_dith.x * ifu_dith.x + ifu_dith.y * ifu_dith.y)
sortidx = np.argsort(dist)
ra_mean = aps.biweight_location(ifu_dith.ra[sortidx][0:2])
dec_mean = aps.biweight_location(ifu_dith.dec[sortidx][0:2])
ixyname = ifu_dith.filename[sortidx][0]
logstr = ''
if '-l' not in rargs:
logstr = '-l %s_%s_%s.log' \
% (args.night, args.shotid,
'_'.join(ixyname.split('_')[0:4]))
f.write('vdrp_calc_flim %s %s %.7f %.7f %s %s %s\n' %
(' '.join(rargs), logstr, ra_mean, dec_mean, args.night,
args.shotid, '_'.join(ixyname.split('_')[0:4])))
# Now prepare the job splitter for it
args.cmdfile = fname
vj.main(args)
def reduce_acam(image):
''' Really simple reduction '''
# Get overscan but skip first column because it looks off
overscan = biweight_location(image[:, 1:4])
# Trim all of the overscan and subtracted the average value
image = image[:, 4:] - overscan
# Subtract the currently global variable "acam_bias" (master bias)
image = image - acam_bias
return image
def chisq_metric(self, chisq, chisq_std_zscore_cut=4.0, return_dict=True):
"""
chi square metrics
chisq : ndarray, shape=(Nants, Ntimes, Nfreqs)
ndarray containing chisq for each antenna (single pol)
chisq_std_cut : float, default=5.0
sigma tolerance (or z-score tolerance) for std of chisq fluctuations
return_dict : bool, default=True
return per-antenna metrics as a dictionary, with antenna number as key
rather than an ndarray
"""
# Get chisq statistics
chisq_avg = np.median(np.median(chisq[:, :, self.band], axis=0),
axis=1).astype(np.float64)
chisq_tot_avg = astats.biweight_location(
chisq[:, :, self.band]).astype(np.float64)
chisq_ant_avg = np.array(
list(map(astats.biweight_location,
chisq[:, :, self.band]))).astype(np.float64)
chisq_ant_std = np.sqrt(
np.array(
list(map(astats.biweight_midvariance, chisq[:, :,
self.band]))))
chisq_ant_std_loc = astats.biweight_location(chisq_ant_std).astype(
np.float64)
chisq_ant_std_scale = np.sqrt(
astats.biweight_midvariance(chisq_ant_std))
chisq_ant_std_zscore = (chisq_ant_std -
chisq_ant_std_loc) / chisq_ant_std_scale
chisq_ant_std_zscore_max = np.max(np.abs(chisq_ant_std_zscore))
chisq_good_sol = chisq_ant_std_zscore_max < chisq_std_zscore_cut
# convert to dictionaries
if return_dict is True:
chisq_ant_std = odict(zip(self.ant_array, chisq_ant_std))
chisq_ant_avg = odict(zip(self.ant_array, chisq_ant_avg))
return (chisq_avg, chisq_tot_avg, chisq_ant_avg, chisq_ant_std,
chisq_ant_std_loc, chisq_ant_std_scale, chisq_ant_std_zscore,
chisq_ant_std_zscore_max, chisq_good_sol)
def build_string(days_ago, mag):
print('Building string...')
data_last24hrs = np.where(days_ago < 1)
data_last1_6_days = np.where((days_ago < 6) & (days_ago > 1))
n_obs_last24hrs = np.size(mag[data_last24hrs])
n_obs_last1_6_days = np.size(mag[data_last1_6_days])
mean_last24hrs = biweight_location(mag[data_last24hrs])
mean_last1_6_days = biweight_location(mag[data_last1_6_days])
stdev = np.std(mag[data_last24hrs]) / np.sqrt(n_obs_last24hrs) \
+ np.std(mag[data_last1_6_days]) / np.sqrt(n_obs_last1_6_days)
diff = mean_last24hrs - mean_last1_6_days
sigma = diff / stdev
if n_obs_last24hrs < 3 or n_obs_last1_6_days < 3:
print('Not enough observations. Abort.')
return None
else:
if diff > 0:
changeword = 'dimmer'
else:
changeword = 'brighter'
mag_text = "My visual mag from last night was " + \
str(format(mean_last24hrs, '.2f')) + \
' (robust mean of ' + \
str(n_obs_last24hrs) + \
' observations). '
change_text = 'That is ' + \
format(abs(diff), '.2f') + \
' mag ' + \
changeword + \
' than the robust mean of the 5 previous nights (n=' + \
str(n_obs_last1_6_days) + \
', ' + \
format(abs(sigma), '.1f') + \
'σ). #Betelgeuse'
text = mag_text + change_text
print(text)
return text
def _stats_data(self, stats, mask, scipy, astropy, decimals_mode):
data = self.data
# The original data size, for computation of valid elements and how
# many are masked/invalid.
size_initial = data.size
# Delete masked values, this will directly convert it to a 1D array
# if the mask is not appropriate then ravel it.
data = data[~mask]
size_masked = data.size
# Delete invalid (NaN, Inf) values. This should ensure that the result
# is always a 1D array
data = data[np.isfinite(data)]
size_valid = data.size
stats['elements'] = [size_valid]
stats['min'] = [np.amin(data)]
stats['max'] = [np.amax(data)]
stats['mean'] = [np.mean(data)]
stats['median'] = [np.median(data)]
# Use custom mode defined in this package because scipy.stats.mode is
# very, very slow and by default tries to calculate the mode along
# axis=0 and not for the whole array.
# Take the first element since the second is the number of occurences.
stats['mode'] = [mode(data, decimals=decimals_mode)[0]]
if astropy:
stats['biweight_location'] = [biweight_location(data)]
stats['std'] = [np.std(data)]
if astropy:
stats['mad'] = [mad_std(data)]
stats['biweight_midvariance'] = [biweight_midvariance(data)]
stats['var'] = [np.var(data)]
if scipy: # pragma: no cover
if not OPT_DEPS['SCIPY']:
log.info('SciPy is not installed.')
else:
# Passing axis=None should not be important since we already
# boolean indexed the array and it's 1D. But it's important
# to remember that there default is axis=0 and not axis=None!
stats['skew'] = [skew(data, axis=None)]
stats['kurtosis'] = [kurtosis(data, axis=None)]
stats['masked'] = [size_initial - size_masked]
stats['invalid'] = [size_masked - size_valid]
return data
def proj_avg(clus_ra,clus_dec,clus_z,gal_ra,gal_dec,gal_z,vlim,rlim,C): # Project Galaxies ang_d,lum_d = C.zdistance(clus_z,H0) angles = C.findangle(gal_ra,gal_dec,clus_ra,clus_dec) rdata = angles * ang_d vdata = c * (gal_z - clus_z) / (1 + clus_z) # Take Average cut = np.where((np.abs(vdata)<vlim)&(rdata<1.5))[0] if len(cut) < 2: cut = np.where((np.abs(vdata)<vlim+500)&(rdata<rlim))[0] if len(cut) < 2: clus_ra = np.median(gal_ra[cut]) clus_dec = np.median(gal_dec[cut]) clus_z = np.median(gal_z[cut]) else: clus_ra = astats.biweight_location(gal_ra[cut]) clus_dec = astats.biweight_location(gal_dec[cut]) clus_z = astats.biweight_location(gal_z[cut]) return clus_ra, clus_dec, clus_z
def proj_avg(clus_ra, clus_dec, clus_z, gal_ra, gal_dec, gal_z, vlim, rlim, C):
# Project Galaxies
ang_d, lum_d = C.zdistance(clus_z, H0)
angles = C.findangle(gal_ra, gal_dec, clus_ra, clus_dec)
rdata = angles * ang_d
vdata = c * (gal_z - clus_z) / (1 + clus_z)
# Take Average
cut = np.where((np.abs(vdata) < vlim) & (rdata < 1.5))[0]
if len(cut) < 2:
cut = np.where((np.abs(vdata) < vlim + 500) & (rdata < rlim))[0]
if len(cut) < 2:
clus_ra = np.median(gal_ra[cut])
clus_dec = np.median(gal_dec[cut])
clus_z = np.median(gal_z[cut])
else:
clus_ra = astats.biweight_location(gal_ra[cut])
clus_dec = astats.biweight_location(gal_dec[cut])
clus_z = astats.biweight_location(gal_z[cut])
return clus_ra, clus_dec, clus_z
def make_plot(days_ago, dates, mag):
print('Making plot...')
time_span = np.max(dates) - np.min(dates)
min_plot = 0
max_plot = +1.5
x_days = 300
# Make bins
bin_width = 1
nights = np.arange(0, max(days_ago), bin_width)
bin_mags = []
errors = []
for night in nights:
selector = np.where((days_ago < night + bin_width)
& (days_ago > night))
n_obs = np.size(mag[selector])
flux = biweight_location(mag[selector])
error = np.std(mag[selector]) / np.sqrt(n_obs)
if error > 0.2:
error = 0
if error == 0: # and flux < 0.2:
flux = np.nan
bin_mags.append(flux)
errors.append(error)
print(night, flux, error, n_obs, np.std(mag[selector]))
# Convert magnitudes to fluxes
bin_mags = np.array(bin_mags)
flux = 1 / (10**(0.4 * (bin_mags - baseline_mag)))
latest_flux = flux[0]
if np.isnan(latest_flux):
latest_flux = flux[1]
plt.errorbar(nights + 0.5, flux, yerr=errors, fmt='.k')
plt.xlabel('Days before today')
plt.ylabel('Normalized flux (0.5 mag baseline)')
plt.ylim(min_plot, max_plot)
plt.xlim(x_days, 0)
date_text = datetime.datetime.now().strftime("%d %b %Y")
try:
lumi = str(int((round(latest_flux * 100, 0))))
text = "#Betelgeuse at " + lumi + r"% of its usual brightness @betelbot "
except:
text = "No new #Betelgeuse brightness tonight @betelbot"
lumi = 0
plt.text(x_days - 2, 0.19, "Update: " + date_text)
plt.text(x_days - 2, 0.12, text)
plt.text(x_days - 2, 0.05, "AAVSO visual (by-eye) daily bins")
plt.savefig(plot_file, bbox_inches='tight', dpi=300)
print('Plot made')
return lumi
def make_plot(days_ago, dates, mag):
print('Making plot...')
time_span = np.max(dates) - np.min(dates)
flatten_lc, trend_lc = flatten(
days_ago,
mag,
method='lowess',
window_length=time_span / 5,
return_trend=True,
)
#mpl.rcParams['font.sans-serif']=['Times New Roman'] #指定默认字体 SimHei为黑体
mpl.rcParams['font.sans-serif'] = ['SimHei']
mpl.rcParams['axes.unicode_minus'] = False #用来正常显示负号
#fontcn = {'family': 'Droid Sans Fallback'} # 1pt = 4/3px
fontcn = {'family': 'SimHei'} # 1pt = 4/3px
fonten = {'family': 'Times New Roman'}
plt.scatter(days_ago, mag, s=5, color='blue', alpha=0.5)
plt.scatter(days_ago1,
all_mags1,
s=10,
color='black',
alpha=0.8,
marker="x")
plt.xlabel(u'从今天往回数的天数', fontdict=fontcn)
#plt.xlabel('从今天往回数的天数')
plt.ylabel(u'视星等', fontdict=fontcn)
mid = 0.6
plt.ylim(mid - 1, mid + 1)
plt.xlim(-1, 20)
plt.plot(days_ago, trend_lc, color='red', linewidth=1)
plt.gca().invert_yaxis()
plt.gca().invert_xaxis()
date_text = datetime.datetime.now().strftime("%Y-%m-%d")
data_last24hrs = np.where(days_ago < 1)
mean_last24hrs = biweight_location(mag[data_last24hrs])
lumi = str(format(mean_last24hrs, '.2f'))
plt.text(19.5, mid + 1 - 0.25, u"裸眼 ", color='blue', fontdict=fontcn)
plt.text(18, mid + 1 - 0.25, u"观测星等 蓝色", color='blue', fontdict=fontcn)
plt.text(14, mid + 1 - 0.25, u"○", color='blue', fontdict=fonten)
plt.text(19.5, mid + 1 - 0.15, u"CCD ", color='black', fontdict=fonten)
plt.text(18, mid + 1 - 0.15, u"观测星等 黑色", color='black', fontdict=fontcn)
plt.text(14, mid + 1 - 0.15, u"×", color='black', fontdict=fonten)
plt.text(19.5, mid + 1 - 0.05, u"局部加权拟合 红色线", color='red', fontdict=fontcn)
plt.text(7.5, mid - 1 + 0.1, u'目前参宿四的星等为 ', fontdict=fontcn)
plt.text(2, mid - 1 + 0.1, lumi, fontdict=fonten)
plt.text(7.5, mid - 1 + 0.2, u"由 天文通 译制于", fontdict=fontcn)
plt.text(3, mid - 1 + 0.2, date_text, fontdict=fonten)
plt.savefig(plot_file, bbox_inches='tight', dpi=300)
print('Done.')
def eval_CMR(self, M1600, beta, keys=None):
lnL = 0.
blob = {}
keys, M1600, beta = self._fix_dim(keys, M1600, beta)
for k, mags, b in zip(keys, M1600, beta):
obsCMR = self.obsData[k]['obsCMR']
obsCMRerr = self.obsData[k]['obsCMRerr']
modelCMR = np.zeros(len(obsCMR))
for iB, (lower, upper) in enumerate(self.obsData[k]['binCMR']):
sample = b[(mags >= lower) & (mags < upper)]
if len(sample) < 2:
modelCMR[iB] = np.nan
else:
modelCMR[iB] = biweight_location(sample)
lnL += self._compute_lnL(modelCMR, obsCMR, obsCMRerr)
blob[k] = modelCMR
return self._retval(lnL, blob)
def calc_background(self, data, axis=None, masked=False):
if self.sigma_clip is not None:
data = self.sigma_clip(data, axis=axis, masked=False)
else:
# convert to ndarray with masked values as np.nan
if isinstance(data, np.ma.MaskedArray):
data = data.filled(np.nan)
# ignore RuntimeWarning where axis is all NaN
with warnings.catch_warnings():
warnings.simplefilter('ignore', RuntimeWarning)
result = biweight_location(data,
c=self.c,
M=self.M,
axis=axis,
ignore_nan=True)
if masked and isinstance(result, np.ndarray):
result = np.ma.masked_where(np.isnan(result), result)
return result
def robustassess(n, d, k=2.24): #raw frequency, doc length and threshold
v = np.divide(n, d)
rawcount = np.sum(n)
mu = biweight_location(v)
s = biweight_scale(v)
mu2s = mu + k * s
vcliph2s = np.minimum(v, mu2s)
#Winsorising
docsclipped = np.sum(np.greater(v, vcliph2s)) # number of docts
adjustedcount = int(np.sum(np.rint(vcliph2s * d))) # rounding to integer
proportionv = n / rawcount # proportion of word frequency per document
proportiond = d / N # document size proportion
kld = np.sum(
np.multiply(proportionv, np.log2(np.divide(proportionv, proportiond))))
return [rawcount, adjustedcount, docsclipped, len(n), '%.3f' % kld]
def _pixel_stats(pixels, clip_sig=3, n_clip=10, min_pix=50):
"""
Calculate mode and scale of pixel distribution, clipping outliers. Uses
biweight as "robust" estimator of these quantities.
:param pixels: Array to calculate statistics for
:param clip_sig: Sigma value at which to clip outliers
:param n_clip: Number of clipping iterations
:param min_pix: Minimum number of retained pixels
:return: 2-tuple of distribution mode, scale
"""
clip_iter = 0
mode, scale = 0, 1
mask = np.ones(pixels.shape, dtype=bool)
while True:
mode = biweight_location(pixels[mask])
scale = biweight_midvariance(pixels[mask])
mask &= np.abs(pixels - mode) < clip_sig * scale
clip_iter += 1
if np.sum(mask) < min_pix or clip_iter >= n_clip:
break
return mode, scale
def calc_background(self, data, axis=None):
if self.sigma_clip is not None:
data = self.sigma_clip(data, axis=axis)
return biweight_location(data, c=self.c, M=self.M, axis=axis)
def overscan_estimate(ccd_in,
meta=None,
master_bias=None,
binsize=None,
min_width=1,
max_width=8,
box_size=100,
min_hist_val=10,
show=False,
*args,
**kwargs):
"""Estimate overscan in ADU in the absense of a formal overscan region
For biases, returns in the median of the image. For all others,
uses the minimum of: (1) the first peak in the histogram of the
image or (2) the minimum of the median of four boxes at the
corners of the image (specific to the IoIO coronagraph)
Works best if bias shape (particularly bias ramp) is subtracted
first. Will subtract bias if bias is supplied and has not been
subtracted.
Parameters
----------
ccd_in : `~astropy.nddata.CCDData` or filename
Image from which to extract overscan estimate
meta : `astropy.io.fits.header` or None
referece to metadata of ccd into which to write OVERSCAN_* cards.
If None, no metadata will be returned
master_bias : `~astropy.nddata.CCDData`, filename, or None
Bias to subtract from ccd before estimate is calculated.
Improves accruacy by removing bias ramp. Bias can be in units
of ADU or electrons and is converted using the specified gain.
If bias has already been subtracted, this step will be skipped
but the bias header will be used to extract readnoise and gain
using the *_key keywords. Default is ``None``.
binsize: float or None, optional
The binsize to use for the histogram. If None, binsize is
(readnoise in ADU)/4. Default = None
min_width : int, optional
Minimum width peak to search for in histogram. Keep in mind
histogram bins are binsize ADU wide. Default = 1
max_width : int, optional
See min_width. Default = 8
box_size : int
Edge size of square box used to extract biweight median location
from the corners of the image for this method of overscan
estimation. Default = 100
show : boolean
Show image with min/max set to highlight overscan pixels and
histogram with overscan chopped histogram. Default is False [consider making this boolean or name of plot file]
"""
if ccd_in.meta.get('overscan_value') is not None:
# Overscan has been subtracted in a previous reduction step,
# so exit quietly
return 0
# Work with a copy since we mess with both the ccd.data and .meta
ccd = ccd_in.copy()
# Get CCD characteristics
ccd.meta = sx694.metadata(ccd.meta)
if meta is None:
meta = ccd.meta
if ccd.unit != u.adu:
# For now don't get fancy with unit conversion
raise ValueError('CCD units must be in ADU for overscan estimation')
if ccd.meta['IMAGETYP'] == "BIAS":
overscan = np.median(ccd)
meta['HIERARCH OVERSCAN_MEDIAN'] = (overscan, 'ADU')
meta['HIERARCH OVERSCAN_METHOD'] = \
('median', 'Method used for overscan estimation')
return overscan
# Prepare for histogram method of overscan estimation. These
# keywords are guaranteed to be in meta because we put there there
# in ccd_metadata
readnoise = ccd.meta['RDNOISE']
gain = ccd.meta['GAIN']
if ccd.meta.get('subtract_bias') is None and master_bias is not None:
# Bias has not been subtracted and we have a bias around to be
# able to do that subtraction
if isinstance(master_bias, str):
bias = CorData.read(master_bias)
meta['HIERARCH OVERSCAN_MASTER_BIAS'] = 'OSBIAS'
meta['OSBIAS'] = master_bias
else:
# Work with a copy since we are going to muck with it
bias = master_bias.copy()
meta['HIERARCH OVERSCAN_MASTER_BIAS'] = 'CCDData object provided'
# Improve our readnoise (measured) and gain (probably not
# re-measured) values
readnoise = bias.meta['RDNOISE']
gain = bias.meta['GAIN']
if bias.unit is u.electron:
# Convert bias back to ADU for subtraction
bias = bias.divide(gain * u.electron / u.adu)
ccd = ccd.subtract(bias)
ccd.meta['HIERARCH subtract_bias'] = True
if type(ccd) != CorData and ccd.meta.get('subtract_bias') is None:
# Don't gunk up logs when we are taking data, but subclasses
# of CorObs (e.g. RedCorObs) will produce message
log.warning(
'overscan_estimate: bias has not been subtracted, which can lead to inaccuracy of overscan estimate'
)
# The coronagraph creates a margin of un-illuminated pixels on the
# CCD. These are great for estimating the bias and scattered
# light for spontanous subtraction.
# Corners method
s = ccd.shape
bs = box_size
c00 = biweight_location(ccd[0:bs, 0:bs])
c10 = biweight_location(ccd[s[0] - bs:s[0], 0:bs])
c01 = biweight_location(ccd[0:bs, s[1] - bs:s[1]])
c11 = biweight_location(ccd[s[0] - bs:s[0], s[1] - bs:s[1]])
corners_method = min(c00, c10, c01, c11)
# Histogram method. The first peak is the bias, the second is the
# ND filter. Note that the 1.25" filters do a better job at this
# than the 2" filters but with carefully chosen parameters, the
# first small peak can be spotted.
if binsize is None:
# Calculate binsize based on readnoise in ADU, but oversample
# by 4. Note need to convert from Quantity to float
binsize = readnoise / gain / 4.
im_hist, im_hist_centers = hist_of_im(ccd, binsize)
# Note that after bias subtraction, there is sometimes some noise
# at low counts. We expect a lot of pixels in the histogram, so filter
good_idx = np.flatnonzero(im_hist > min_hist_val)
im_hist = im_hist[good_idx]
im_hist_centers = im_hist_centers[good_idx]
# The arguments to linspace are the critical parameters I played
# with together with binsize to get the first small peak to be recognized
im_peak_idx = signal.find_peaks_cwt(im_hist,
np.linspace(min_width, max_width))
hist_method = im_hist_centers[im_peak_idx[0]]
overscan_methods = ['corners', 'histogram']
overscan_values = np.asarray((corners_method, hist_method))
meta['HIERARCH OVERSCAN_CORNERS'] = (corners_method, 'ADU')
meta['HIERARCH OVERSCAN_HISTOGRAM'] = (hist_method, 'ADU')
o_idx = np.argmin(overscan_values)
overscan = overscan_values[o_idx]
meta['HIERARCH OVERSCAN_METHOD'] = (overscan_methods[o_idx],
'Method used for overscan estimation')
if show:
fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(8.5, 9))
ccds = ccd.subtract(1000 * u.adu)
range = 5 * readnoise / gain
vmin = overscan - range - 1000
vmax = overscan + range - 1000
ax1.imshow(ccds,
origin='lower',
cmap=plt.cm.gray,
filternorm=0,
interpolation='none',
vmin=vmin,
vmax=vmax)
ax1.set_title('Image minus 1000 ADU')
ax2.plot(im_hist_centers, im_hist)
ax2.set_yscale("log")
ax2.set_xscale("log")
ax2.axvline(overscan, color='r')
# https://stackoverflow.com/questions/13413112/creating-labels-where-line-appears-in-matplotlib-figure
# the x coords of this transformation are data, and the
# y coord are axes
trans = transforms.blended_transform_factory(ax2.transData,
ax2.transAxes)
ax2.set_title('Histogram')
ax2.text(overscan + 20,
0.05,
overscan_methods[o_idx] +
' overscan = {:.2f}'.format(overscan),
rotation=90,
transform=trans,
verticalalignment='bottom')
plt.show()
return overscan
def biweight_location(self):
"""
The biweight location of the pixel values.
"""
return biweight_location(self.goodvals)
def calc_background(self, data):
return biweight_location(self.sigma_clip(data), c=self.c, M=self.M)
def fbimean(y):
"""Helper function of trendplot. Returns biweight mean of input 1d-array (see astropy.stats)"""
return biweight_location(y)
def calc_background(self, data, axis=None):
if self.sigma_clip is not None:
data = self.sigma_clip(data, axis=axis)
return biweight_location(data, c=self.c, M=self.M, axis=axis)
clus_dec = DEC[i]
clus_z = Z[i]
# Load Galaxies
galdata = C4.load_chris_gals(HaloID[i])
gal_ra,gal_dec,gal_z,gal_gmags,gal_rmags,gal_imags = galdata
# Project Galaxies
ang_d,lum_d = C.zdistance(clus_z,H0)
angles = C.findangle(gal_ra,gal_dec,clus_ra,clus_dec)
rdata = angles * ang_d
vdata = c * (gal_z - clus_z) / (1 + clus_z)
# Take Average Three times
cut1 = np.where((np.abs(vdata)<1000)&(rdata<1.5))[0]
clus_ra = astats.biweight_location(gal_ra[cut1])
clus_dec = astats.biweight_location(gal_dec[cut1])
clus_z = astats.biweight_location(gal_z[cut1])
ang_d,lum_d = C.zdistance(clus_z,H0)
angles = C.findangle(gal_ra,gal_dec,clus_ra,clus_dec)
rdata = angles * ang_d
vdata = c * (gal_z - clus_z) / (1 + clus_z)
cut2 = np.where((np.abs(vdata)<500)&(rdata<1.5))[0]
clus_ra = astats.biweight_location(gal_ra[cut2])
clus_dec = astats.biweight_location(gal_dec[cut2])
clus_z = astats.biweight_location(gal_z[cut2])
ang_d,lum_d = C.zdistance(clus_z,H0)
angles = C.findangle(gal_ra,gal_dec,clus_ra,clus_dec)
def return_biweight_one_sigma(shots, pixlo=5, pixhi=25, wave_bins=None):
"""
Return the biweight midvariance of the
1 sigma values of a series of shots. Trim
off edge values
Parameters
----------
shots : list
a list of shots to loop
over
pixlo, pixhi : int (optional)
limits for the 2D ra/dec
pixel array when computing
the values. Default is 5,25.
"""
sigmas_all = []
for shot in shots:
fn, fnmask = return_sensitivity_hdf_path(shot,
return_mask_fn = True)
print(fn)
with SensitivityCubeHDF5Container(fn, mask_filename = fnmask) as hdf:
first = True
for ifuslot, scube in hdf.itercubes():
# assume wavelenghth WCS the same for whole HDF
if (type(wave_bins) != type(None)) and first:
junkx, junky, wlo = scube.wcs.all_world2pix(0., 0., wave_bins[:-1], 0)
junkx, junky, whi = scube.wcs.all_world2pix(0., 0., wave_bins[1:], 0)
first = False
maxsize = int(max(floor(whi) - ceil(wlo))*(pixhi - pixlo)*(pixhi - pixlo))
if type(wave_bins) != type(None):
sigmas = []
for lo, high in zip(wlo, whi):
sigmas_slice = scube.sigmas.filled(nan)[int(ceil(lo)):int(floor(high)),
pixlo:pixhi, pixlo:pixhi]
sigmas_slice = sigmas_slice.reshape(sigmas_slice.shape[0]*sigmas_slice.shape[1]*sigmas_slice.shape[2])
# pad with NaN to keep consistent size
if len(sigmas_slice) < maxsize:
sigmas_slice = pad(sigmas_slice, (0, maxsize - len(sigmas_slice)),
'empty')
sigmas.append(sigmas_slice)
else:
sigmas = scube.sigmas.filled(nan)[:, pixlo:pixhi, pixlo:pixhi]
sigmas = sigmas.reshape(sigmas.shape[0], sigmas.shape[1]*sigmas.shape[2])
sigmas_all.extend(array(sigmas).T)
# assume wavelenghth WCS the same for whole HDF
if type(wave_bins) == type(None):
pixels = arange(scube.sigmas.shape[0])
junkras, junkdecs, waves = scube.wcs.all_pix2world(0*pixels, 0*pixels,
pixels, 0)
else:
waves = 0.5*(wave_bins[1:] + wave_bins[:-1])
biwt = biweight_location(array(sigmas_all), axis=0, ignore_nan=True)
return waves, biwt
def na_back_directory(directory,
calibration=True,
read_pout=True,
write_pout=True,
write_plot=True,
create_outdir=True,
show=False,
**kwargs):
rd = reduced_dir(directory, create=False)
poutname = os.path.join(rd, 'Na_back.pout')
pout = cached_pout(na_back_pipeline,
poutname=poutname,
read_pout=read_pout,
write_pout=write_pout,
create_outdir=create_outdir,
directory=directory,
**kwargs)
if len(pout) == 0:
log.debug(f'no Na background measurements found in {directory}')
return {}
_, pipe_meta = zip(*pout)
na_back_list = [pm['Na_back'] for pm in pipe_meta]
df = pd.DataFrame(na_back_list)
df.sort_values('jd')
just_date = df['date'].iloc[0]
instr_mag = u.Magnitude(df['best_back'] * u.electron / u.s / u.pix**2)
df['instr_mag'] = instr_mag
#tdf = df.loc[df['airmass'] < 2.0]
tdf = df.loc[df['airmass'] < 2.5]
mean_back = np.mean(tdf['best_back'])
std_back = np.std(tdf['best_back'])
biweight_back = biweight_location(tdf['best_back'])
mad_std_back = mad_std(tdf['best_back'])
# https://stackoverflow.com/questions/20664980/pandas-iterate-over-unique-values-of-a-column-that-is-already-in-sorted-order
# and
# https://pandas.pydata.org/pandas-docs/stable/user_guide/groupby.html
#https://matplotlib.org/stable/tutorials/intermediate/color_cycle.html
offset_cycler = cycler(color=['r', 'g', 'b', 'y'])
plt.rc('axes', prop_cycle=offset_cycler)
f = plt.figure(figsize=[8.5, 11])
plt.suptitle(f"Na background {just_date}")
offset_groups = df.groupby(['raoff', 'decoff']).groups
ax = plt.subplot(3, 1, 1)
for offset_idx in offset_groups:
gidx = offset_groups[offset_idx]
gdf = df.iloc[gidx]
plot_dates = julian2num(gdf['jd'])
plt.plot_date(plot_dates,
gdf['best_back'],
label=f"dRA {gdf.iloc[0]['raoff']} "
f"dDEC {gdf.iloc[0]['decoff']} armin")
plt.axhline(y=biweight_back, color='red')
plt.axhline(y=biweight_back + mad_std_back,
linestyle='--',
color='k',
linewidth=1)
plt.axhline(y=biweight_back - mad_std_back,
linestyle='--',
color='k',
linewidth=1)
plt.text(0.5,
biweight_back + 0.1 * mad_std_back,
f'{biweight_back:.4f} +/- {mad_std_back:.4f}',
ha='center',
transform=ax.get_yaxis_transform())
plt.xlabel('date')
plt.ylabel('electron/s')
ax.legend()
ax = plt.subplot(3, 1, 2)
for offset_idx in offset_groups:
gidx = offset_groups[offset_idx]
gdf = df.iloc[gidx]
plt.plot(gdf['airmass'], gdf['instr_mag'], '.')
#plt.axhline(y=biweight_back, color='red')
#plt.axhline(y=biweight_back+mad_std_back,
# linestyle='--', color='k', linewidth=1)
#plt.axhline(y=biweight_back-mad_std_back,
# linestyle='--', color='k', linewidth=1)
plt.xlabel('Airmass')
plt.ylabel('mag (electron/s/pix^2')
ax = plt.subplot(3, 1, 3)
for offset_idx in offset_groups:
gidx = offset_groups[offset_idx]
gdf = df.iloc[gidx]
plt.plot(gdf['alt'], gdf['best_back'], '.')
plt.axhline(y=biweight_back, color='red')
plt.axhline(y=biweight_back + mad_std_back,
linestyle='--',
color='k',
linewidth=1)
plt.axhline(y=biweight_back - mad_std_back,
linestyle='--',
color='k',
linewidth=1)
plt.xlabel('Alt')
plt.ylabel('electron/s')
f.subplots_adjust(hspace=0.3)
if write_plot is True:
write_plot = os.path.join(rd, 'Na_back.png')
if isinstance(write_plot, str):
plt.savefig(write_plot, transparent=True)
if show:
plt.show()
plt.close()
# Problem discussed in https://mail.python.org/pipermail/tkinter-discuss/2019-December/004153.html
gc.collect()
return {
'date': just_date,
'jd': np.floor(df['jd'].iloc[0]),
'biweight_back': biweight_back,
'mad_std_back': mad_std_back,
'na_back_list': na_back_list
}
def richness_est(self,ra,dec,z,pz,gmags,rmags,imags,abs_rmags,haloid,clus_ra,clus_dec,clus_z,clus_rvir=None,gr_slope=None,gr_width=None,gr_inter=None,shiftgap=True,use_specs=False,use_bcg=False,fit_rs=False,spec_list=None,fixed_aperture=False,fixed_vdisp=False,plot_gr=False,plot_sky=False,plot_phase=False,find_pairs=True):
''' Richness estimator, magnitudes should be apparent mags except for abs_rmag
z : spectroscopic redshift
pz : photometric redshift
gr_slope : if fed color-mag (g-r vs. r) slope of fit to red sequence
gr_width : if fed color-mag (g-r vs. r) width of fit to red sequence
gr_inter : if red color-mag (g-r vs. r) intercept of fit to red sequence
spec_list : indexing array specifying gals w/ spectroscopic redshifts, preferably fed as boolean numpy array
fixed_aperture : clus_rvir = 1 Mpc
fixed_vdisp : clus_vdisp = 1000 km/s (members < clus_vdisp*2)
use_specs : includes all spectro members in richness regardless of their color, subject to counting twice
use_bcg : use bcg RS_color as cluster RS_color
fit_rs : fit a line to the member galaxy RS relationship, as of now this does not work and we take a flat RS relationship
plot_gr : plot r vs g-r color diagram with RS fits
plot_sky : plot RA and DEC of gals with signal and background annuli
plot_phase : plot rdata and vdata phase space
find_pairs : run c4 cluster pair identifier on phase spaces
- If at least one quarter of the outer annuli doesn't have any galaxies (perhaps because it has run over the observation edge),
it will return -99 as a richness
'''
# Put Into Class Namespace
keys = ['ra','dec','z','pz','gmags','rmags','imags','clus_ra','clus_dec','clus_z','clus_rvir','vel_disp','color_data','color_cut','RS_color','RS_sigma','clus_color_cut','signal','background','richness','mems','phot','spec','spec_mems','deg_per_rvir','SA_outer','SA_inner','outer_red_dense','inner_background','Nphot_inner','Nphot_outer','red_inner','red_outer','all','outer_edge','inner_edge','shift_cut','set','pair','Nspec','gr_slope','gr_inter','gr_width']
self.__dict__.update(ez.create(keys,locals()))
# Take rough virial radius measurement, this method is BAD...
if clus_rvir == None:
clus_rvir = np.exp(-1.86)*len(np.where((rmags < -19.55) & (rdata < 1.0) & (np.abs(vdata) < 3500))[0])**0.51
self.clus_rvir = clus_rvir
# Set clus_rvir = 1.0 Mpc if fixed aperture == True
if fixed_aperture == True:
clus_rvir = 1.0
# Magnitue Cut at 19.1 Apparent R Mag, then at -19 Absolute R Mag and sort by them
bright = np.where(rmags < 19.1)[0]
data = np.vstack([ra,dec,z,pz,gmags,rmags,imags,abs_rmags])
data = data.T[bright].T
ra,dec,z,pz,gmags,rmags,imags,abs_rmags = data
bright = np.where(abs_rmags < -19.5)[0]
data = np.vstack([ra,dec,z,pz,gmags,rmags,imags,abs_rmags])
data = data.T[bright].T
ra,dec,z,pz,gmags,rmags,imags,abs_rmags = data
sorts = np.argsort(abs_rmags)
data = np.vstack([ra,dec,z,pz,gmags,rmags,imags,abs_rmags])
data = data.T[sorts].T
ra,dec,z,pz,gmags,rmags,imags,abs_rmags = data
self.__dict__.update(ez.create(keys,locals()))
# Separate Into Spectro Z and Photo Z DataSets
# Define the Spectro Z catalogue
if spec_list == None:
spec_cut = np.abs(z) > 1e-4
else:
if type(spec_list) == list: spec_list = np.array(spec_list)
if spec_list.dtype != 'bool':
spec_cut = np.array([False]*len(ra))
spec_cut[spec_list] = True
all = {'ra':ra,'dec':dec,'z':z,'pz':pz,'gmags':gmags,'rmags':rmags,'imags':imags,'abs_rmags':abs_rmags}
spec = {'ra':ra[spec_cut],'dec':dec[spec_cut],'z':z[spec_cut],'pz':pz[spec_cut],'gmags':gmags[spec_cut],'rmags':rmags[spec_cut],'imags':imags[spec_cut],'abs_rmags':abs_rmags[spec_cut]}
phot = {'ra':ra[~spec_cut],'dec':dec[~spec_cut],'z':z[~spec_cut],'pz':pz[~spec_cut],'gmags':gmags[~spec_cut],'rmags':rmags[~spec_cut],'imags':imags[~spec_cut],'abs_rmags':abs_rmags[~spec_cut]}
# Project Spectra into radius and velocity
ang_d,lum_d = C.zdistance(clus_z,self.H0)
angles = C.findangle(spec['ra'],spec['dec'],clus_ra,clus_dec)
rdata = angles * ang_d
vdata = self.c * (spec['z'] - clus_z) / (1 + clus_z)
spec.update({'rdata':rdata,'vdata':vdata})
self.__dict__.update(ez.create(keys,locals()))
# Take Hard Phasespace Limits
limit = np.where( (rdata < 5) & (np.abs(vdata) < 5000) )[0]
clus_data = np.vstack([rdata,vdata,spec['ra'],spec['dec'],spec['z'],spec['pz'],spec['gmags'],spec['rmags'],spec['imags'],spec['abs_rmags']])
clus_data = clus_data.T[limit].T
rdata,vdata,spec['ra'],spec['dec'],spec['z'],spec['pz'],spec['gmags'],spec['rmags'],spec['imags'],spec['abs_rmags'] = clus_data
spec.update({'rdata':rdata,'vdata':vdata})
self.__dict__.update(ez.create(keys,locals()))
# Shiftgapper for Interlopers
if shiftgap == True:
len_before = np.where((rdata < clus_rvir*1.5)&(np.abs(vdata)<4000))[0].size
clus_data = np.vstack([rdata,vdata,spec['ra'],spec['dec'],spec['z'],spec['pz'],spec['gmags'],spec['rmags'],spec['imags'],spec['abs_rmags']])
clus_data = C.shiftgapper(clus_data.T).T
sorts = np.argsort(clus_data[-1])
clus_data = clus_data.T[sorts].T
rdata,vdata,spec['ra'],spec['dec'],spec['z'],spec['pz'],spec['gmags'],spec['rmags'],spec['imags'],spec['abs_rmags'] = clus_data
shift_cut = len_before - np.where((rdata < clus_rvir)&(np.abs(vdata)<4000))[0].size
# Measure Velocity Dispersion of all galaxies within 1 * r_vir and np.abs(vdata) < 4000 km/s
spec.update({'rdata':rdata,'vdata':vdata})
self.__dict__.update(ez.create(keys,locals()))
vel_disp = astats.biweight_midvariance(vdata[np.where((rdata < clus_rvir*1)&(np.abs(vdata) < 4000))])
if fixed_vdisp == True: vel_disp = 1000
# Run Cluster Pair Finder
if find_pairs == True:
pair, d1_chi, d2_chi, d3_chi, s_chi, double1, double2, double3, single, v_range, bins = pair_find(rdata,vdata)
# Calculate Nspec, Nspec = # of galaxies within RVIR
try:
Nspec = np.where(rdata<clus_rvir)[0].size
except:
Nspec = 0
self.__dict__.update(ez.create(keys,locals()))
# Get Members from Specta, get their red sequence color
mems = np.where((spec['rdata'] < clus_rvir)&(np.abs(spec['vdata'])<2*vel_disp))[0]
color_data = spec['gmags'] - spec['rmags']
color_cut = np.where((color_data[mems] < 1.2) & (color_data[mems] > 0.65))[0]
RS_color = astats.biweight_location(color_data[mems][color_cut])
RS_shift = color_data[mems][color_cut] - RS_color
RS_sigma = astats.biweight_midvariance(RS_shift[np.where(np.abs(RS_shift)<.15)])
if fit_rs == True:
clf = linear_model.LinearRegression()
set = np.where(np.abs(color_data[mems]-RS_color)<3*RS_sigma)[0]
clf.fit(spec['rmags'][mems][set].reshape(set.size,1),color_data[mems][set])
if use_bcg == True:
bright = np.argsort(all['abs_rmags'][mems])[0]
RS_color = color_data[mems][bright]
RS_shift = color_data[mems][color_cut] - RS_color
RS_sigma = astats.biweight_midvariance(RS_shift[np.where(np.abs(RS_shift)<.15)])
# spec_mems is # of members that are within 2 sigma of cluster color
clus_color_cut = np.where(np.abs(color_data[mems] - RS_color) < RS_sigma*2)[0]
spec_mems = len(clus_color_cut)
self.__dict__.update(ez.create(keys,locals()))
# If fed gr fit
if gr_slope != None:
def RS_color(r_mag): return r_mag*gr_slope + gr_inter
RS_sigma = gr_width / 2
clus_color_cut = np.where(np.abs(color_data[mems] - RS_color(spec['rmags'])[mems]) < RS_sigma*2)[0]
spec_mems = len(clus_color_cut)
self.__dict__.update(ez.create(keys,locals()))
# Get Rdata from PhotoZ & SpecZ Data Set
angles = C.findangle(all['ra'],all['dec'],clus_ra,clus_dec)
all['rdata'] = angles * ang_d
# Get deg per RVIR proper
deg_per_rvir = R.Cosmo.arcsec_per_kpc_proper(clus_z).value * 1e3 * clus_rvir / 3600
# Get Area of Virial Circle out to 1 rvir, and Area of outer annuli, which is annuli from 4rvir < R < 6rvir, or dependent on rvir
if clus_rvir < 2.5:
outer_edge = 6.0
inner_edge = 4.0
elif clus_rvir >= 2.5 and clus_rvir < 3:
outer_edge = 5.0
inner_edge = 3.5
else:
outer_edge = 3.0
inner_edge = 2.0
SA_inner = np.pi*deg_per_rvir**2
SA_outer = np.pi * ( (outer_edge*deg_per_rvir)**2 - (inner_edge*deg_per_rvir)**2 )
# Get Number of Cluster Color Galaxies from Photo Data Set in Inner Circle and Outer Annuli
RS_color_sig = 2.0
if gr_slope == None:
red_inner = np.where(((all['gmags']-all['rmags']) < RS_color + RS_color_sig*RS_sigma)&((all['gmags']-all['rmags']) > RS_color - RS_color_sig*RS_sigma)&(all['rdata']<clus_rvir))[0]
red_outer = np.where(((all['gmags']-all['rmags']) < RS_color + 1.5*RS_sigma)&((all['gmags']-all['rmags']) > RS_color - 1.5*RS_sigma)&(all['rdata']<outer_edge*clus_rvir)&(all['rdata']>inner_edge*clus_rvir))[0]
else:
red_inner = np.where(((all['gmags']-all['rmags']) < RS_color(all['rmags']) + RS_color_sig*RS_sigma)&((all['gmags']-all['rmags']) > RS_color(all['rmags']) - RS_color_sig*RS_sigma)&(all['rdata']<clus_rvir))[0]
red_outer = np.where(((all['gmags']-all['rmags']) < RS_color(all['rmags']) + 1.5*RS_sigma)&((all['gmags']-all['rmags']) > RS_color(all['rmags']) - 1.5*RS_sigma)&(all['rdata']<outer_edge*clus_rvir)&(all['rdata']>inner_edge*clus_rvir))[0]
Nphot_inner = len(red_inner)
Nphot_outer = len(red_outer)
self.__dict__.update(ez.create(keys,locals()))
# Get Solid Angle Density of Outer Red Galaxies
outer_red_dense = Nphot_outer / SA_outer
inner_background = int(np.ceil(outer_red_dense * SA_inner))
# If inner_background is less than Nphot_inner, then Nphot_inner -= inner_background, otherwise Nphot_inner = 0
if inner_background < Nphot_inner:
Nphot_inner -= inner_background
else:
Nphot_inner = 0
# Richness = spec_mems + Nphot_inner or just Nphot_inner
if use_specs == True:
richness = spec_mems + Nphot_inner
else:
richness = Nphot_inner
self.__dict__.update(ez.create(keys,locals()))
# Plot
if plot_gr == True:
fig,ax = mp.subplots()
ax.plot(rmags,gmags-rmags,'ko',alpha=.8)
ax.plot(spec['rmags'][mems],color_data[mems],'co')
if gr_slope == None:
ax.axhline(RS_color,color='r')
ax.axhline(RS_color+RS_sigma,color='b')
ax.axhline(RS_color-RS_sigma,color='b')
else:
xdata = np.arange(spec['rmags'][mems].min(),spec['rmags'][mems].max(),.1)
ax.plot(xdata,RS_color(xdata),color='r')
ax.plot(xdata,RS_color(xdata)+RS_sigma,color='b')
ax.plot(xdata,RS_color(xdata)-RS_sigma,color='b')
ax.set_xlim(13,19)
ax.set_ylim(0,1.3)
ax.set_xlabel('Apparent R Mag',fontsize=16)
ax.set_ylabel('App G Mag - App R Mag',fontsize=16)
ax.set_title('Color-Mag Diagram, Cluster '+str(haloid))
fig.savefig('colormag_'+str(haloid)+'.png',bbox_inches='tight')
mp.close(fig)
if plot_sky == True:
fig,ax = mp.subplots()
ax.plot(all['ra'],all['dec'],'ko')
ax.plot(all['ra'][red_inner],all['dec'][red_inner],'ro')
ax.plot(all['ra'][red_outer],all['dec'][red_outer],'yo')
ax.plot(clus_ra,clus_dec,'co',markersize=9)
ax.set_xlabel('RA',fontsize=15)
ax.set_ylabel('Dec.',fontsize=15)
ax.set_title('Richness Annuli for Halo '+str(haloid))
fig.savefig('skyplot_'+str(haloid)+'.png',bbox_inches='tight')
mp.close(fig)
if plot_phase == True:
fig,ax = mp.subplots()
ax.plot(spec['rdata'],spec['vdata'],'ko')
ax.plot(spec['rdata'][mems],spec['vdata'][mems],'co')
bcg = np.where(spec['abs_rmags']==spec['abs_rmags'][mems].min())[0][0]
ax.plot(spec['rdata'][bcg],spec['vdata'][bcg],'ro')
ax.set_xlim(0,5)
ax.set_ylim(-5000,5000)
ax.set_xlabel('Radius (Mpc)',fontsize=15)
ax.set_ylabel('Velocity (km/s)',fontsize=15)
ax.set_title('phasespace haloid '+str(haloid))
fig.savefig('phasespace_'+str(haloid)+'.png',bbox_inches='tight')
mp.close(fig)
# Check to make sure galaxies exist (somewhat) uniformely in outer annuli (aka. cluster isn't on edge of observation strip)
# First, project all galaxies into polar coordinates centered on cluster center
x = (all['ra']-clus_ra)/np.cos(clus_dec*np.pi/180) # x coordinate in arcsec (of dec) centered on cluster center
y = (all['dec']-clus_dec)
all['radius'] = np.sqrt( x**2 + y**2 ) / deg_per_rvir #radius scaled by RVIR
all['theta'] = np.arctan( y / x )
# Add corrections to arctan function
all['theta'][np.where( (x < 0) & (y > 0) )] += np.pi # Quadrant II
all['theta'][np.where( (x < 0) & (y < 0) )] += np.pi # Quadrant III
all['theta'][np.where( (x > 0) & (y < 0) )] += 2*np.pi # Quadrant IV
# Then break outer annuli into 4 sections and check if at least 1 galaxy exists in each section
sizes1 = np.array([np.where((np.abs(all['theta']-i)<=np.pi/2)&(all['radius']>inner_edge)&(all['radius']<15))[0].size for i in np.linspace(0,2*np.pi,4)])
# Do it again but shift theta by np.pi/8 this time
sizes2 = np.array([np.where((np.abs(all['theta']-i)<=np.pi/2)&(all['radius']>inner_edge)&(all['radius']<15))[0].size for i in np.linspace(np.pi/8,17*np.pi/8,4)])
if 0 in sizes1 or 0 in sizes2:
mp.plot(all['radius'],all['theta'],'ko')
mp.xlabel('radius')
mp.ylabel('theta')
mp.savefig('rth_'+str(haloid)+'.png')
mp.close()
print 'sizes1=',sizes1
print 'sizes2=',sizes2
return -99
return richness
def calc_stat(data, sigma=1.8, niter=10, algorithm='median'):
"""Calculate statistics for given data.
Parameters
----------
data : ndarray
Data to be calculated from.
sigma : float
Sigma for sigma clipping.
niter : int
Number of iterations for sigma clipping.
algorithm : {'mean', 'median', 'mode', 'stddev'}
Algorithm for statistics calculation.
Returns
-------
val : float
Statistics value.
Raises
------
ValueError
Invalid algorithm.
"""
arr = np.ravel(data)
if len(arr) < 1:
return 0.0
# NOTE: Now requires Astropy 1.1 or later, so this check is not needed.
# from astropy import version as astropy_version
# if ((astropy_version.major==1 and astropy_version.minor==0) or
# (astropy_version.major < 1)):
# arr_masked = sigma_clip(arr, sig=sigma, iters=niter)
# else:
# arr_masked = sigma_clip(arr, sigma=sigma, iters=niter)
arr_masked = sigma_clip(arr, sigma=sigma, iters=niter)
arr = arr_masked.data[~arr_masked.mask]
if len(arr) < 1:
return 0.0
algorithm = algorithm.lower()
if algorithm == 'mean':
val = arr.mean()
elif algorithm == 'median':
val = np.median(arr)
elif algorithm == 'mode':
val = biweight_location(arr)
elif algorithm == 'stddev':
val = arr.std()
else:
raise ValueError('{0} is not a valid algorithm for sky background '
'calculations'.format(algorithm))
return val
header=hdu_counts[1].header))
# if there's more than one filter, do reprojection
if len(filter_list) > 1:
for f in range(1,len(filter_list)):
new_array, _ = reproject_interp(hdu_list[f], hdu_list[0].header)
hdu_list[f] = fits.ImageHDU(data=new_array, header=hdu_list[0].header)
# normalize the images
for f in range(len(filter_list)):
# subtract mode
# - do a sigma clip
pix_clip = sigma_clip(hdu_list[f].data, sigma=2.5, iters=3)
# - calculate biweight
biweight_clip = biweight_location(pix_clip.data[~pix_clip.mask])
# - subtraction
new_array = hdu_list[f].data - biweight_clip
# set anything below 0 to 0
new_array[new_array < 0] = 0
# set 95th percentile to 1
new_array = new_array/np.nanpercentile(new_array, 95)
# save it
hdu_list[f].data = new_array
# add the images together
im_sum = np.mean([hdu_list[f].data for f in range(len(filter_list))], axis=0)
