def raw_flux_plot(target, phot_type='aper'):
pines_path = pines_dir_check()
short_name = short_name_creator(target)
phot_path = pines_path / ('Objects/' + short_name + '/' + phot_type +
'_phot/')
phot_files = natsorted(glob(str(phot_path / '*.csv')))
for i in range(len(phot_files)):
phot_file = phot_files[i]
df = pd.read_csv(phot_file)
df.columns = df.keys().str.strip()
times = np.array(df['Time JD'])
sources = natsorted(
list(
set([
i.split(' ')[0] + ' ' + i.split(' ')[1] for i in df.keys()
if (i[0] == '2') or (i[0] == 'R')
])))
num_sources = len(sources)
#Unnormalized flux plot
fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(10, 5), sharex=True)
for j in range(num_sources):
source_name = sources[j]
flux = df[source_name + ' Flux']
if j == 0:
ls = '-'
else:
ls = ''
ax.plot(times, flux, linestyle=ls, marker='.')
ax.set_title('Raw flux')
#Median-normalized flux plot
fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(10, 5), sharex=True)
for j in range(num_sources):
source_name = sources[j]
flux = df[source_name + ' Flux'] / np.median(
df[source_name + ' Flux'])
if j == 0:
ls = '-'
else:
ls = ''
ax.plot(times, flux, linestyle=ls, marker='.')
ax.set_title('Median-normalized flux')
pdb.set_trace()
#Mean-normalized flux plot
fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(10, 5), sharex=True)
for j in range(num_sources):
source_name = sources[j]
flux = df[source_name + ' Flux'] / np.mean(
df[source_name + ' Flux'])
if j == 0:
ls = '-'
else:
ls = ''
ax.plot(times, flux, linestyle=ls, marker='.')
ax.set_title('Median-normalized flux')
pdb.set_trace()
def hot_pixels(date, exptime, box_l=3, saturation=4000, upload=False, sftp=''):
if box_l % 2 == 0:
raise ValueError('box_l must be odd!')
pines_path = pines_dir_check()
darks_path = pines_path / ('Calibrations/Darks/Master Darks/')
all_dark_stddev_files = natsort.natsorted(
list(Path(darks_path).rglob('*' + date + '.fits')))
dark_file = ''
for file in all_dark_stddev_files:
if float(file.name.split('_')[2]) == exptime:
dark_file = file
if dark_file == '':
raise RuntimeError('No dark stddev files found on disk with date ' +
date + ' and exposure time ' + str(exptime) +
' seconds!')
master_dark = fits.open(dark_file)[0].data
shape = np.shape(master_dark)
avg, med, std = sigma_clipped_stats(master_dark)
hot_pixel_mask = np.zeros((shape[0], shape[1]), dtype='int')
#Mask/nan any pixel that is > saturation in the master_dark
hot_pixel_mask[np.where(master_dark > saturation)] = 1
master_dark[np.where(master_dark > saturation)] = np.nan
#Incorporate bad pixel information from the variable/Kokopelli pixel masks.
# variable_path = pines_path/('Calibrations/Variable Pixel Masks/vpm_'+str(exptime)+'_s_'+date+'.fits')
# variable_mask = fits.open(variable_path)[0].data
# master_dark[np.where(variable_mask == 1)] = np.nan
# kokopelli_path = pines_path/('Calibrations/Kokopelli Mask/kokopelli_mask.fits')
# kokopelli_mask = (1-fits.open(kokopelli_path)[0].data).astype('int')[0:1024,:]
# master_dark[np.where(kokopelli_mask == 1)] = np.nan
print('')
print('Flagging hot pixels.')
#This will iterate until it finds no more hot pixels.
total_flagged = 0
clip_lvls = [10, 10, 10, 9, 8]
box_ls = [13, 11, 9, 7, 5]
iteration = 1
for i in range(len(box_ls)):
box_l = box_ls[i]
clip_lvl = clip_lvls[i]
print('Box size = {} x {}.'.format(box_l, box_l))
print('Sigma clipping level = {}.'.format(clip_lvl))
print('......')
num_flagged = 999 #Initialize. Keep iterating until num_flagged = 1 or 0.
box_minus = int(box_l / 2)
box_plus = int(box_l / 2) + 1
targ_pix_ind = int(box_l**2 / 2)
while (num_flagged > 1):
num_flagged = 0
pbar = ProgressBar()
for xx in pbar(range(int(box_l / 2), shape[0] - int(box_l / 2))):
for yy in range(int(box_l / 2), shape[1] - int(box_l / 2)):
if (not np.isnan(master_dark[yy, xx])):
box_2d = master_dark[
yy - box_minus:yy + box_plus, xx - box_minus:xx +
box_plus] #2d cutout surrounding the target pixel.
box_1d = box_2d[~np.isnan(box_2d)].ravel(
) #Unraveled box, ignoring any NaNs.
try:
neighbor_vals = np.delete(
box_1d, targ_pix_ind
) #Remove the target pixel from the 1d array.
#neighbor_vals = sigmaclip(n, low=3.5, high=3.5)[0] #Clip away other bad pixels in the box to avoid biasing the mean/standard deviation.
if master_dark[yy, xx] > np.median(
neighbor_vals
) + clip_lvl * np.std(
neighbor_vals
): #Flag pixel as hot if it is more than clip_lvl sigma higher than the mean of its neighbors.
hot_pixel_mask[yy, xx] = 1
master_dark[
yy,
xx] = np.nan #Set this pixel to a NaN so that it's ignored on subsequent iterations.
num_flagged += 1
except:
continue
iteration += 1
total_flagged += num_flagged
print(
'Iteration {}: {} new hot pixels identified, {} hot pixels total.'
.format(iteration, num_flagged, total_flagged))
print('')
print('')
print('Found {} hot pixels.'.format(total_flagged))
output_filename = 'hpm_' + str(exptime) + '_s_' + date + '.fits'
output_path = pines_path / ('Calibrations/Hot Pixel Masks/' +
output_filename)
#Add some header keywords detailing the master_dark creation process.
hdu = fits.PrimaryHDU(hot_pixel_mask)
hdu.header['HIERARCH DATE CREATED'] = datetime.utcnow().strftime(
'%Y-%m-%d') + 'T' + datetime.utcnow().strftime('%H:%M:%S')
hdu.header['HIERARCH SIGMA CLIP LVL'] = clip_lvl
#Now save to a file on your local machine.
print('')
print('Writing the file to ' + output_filename)
# #Check to see if other files of this name exist.
# if os.path.exists(output_path):
# print('')
# print('WARNING: This will overwrite {}!'.format(output_path))
# dark_check = input('Do you want to continue? y/n: ')
# if dark_check == 'y':
# hdu.writeto(output_path,overwrite=True)
# print('Wrote to {}!'.format(output_path))
# else:
# print('Not overwriting!')
# else:
print('Wrote to {}!'.format(output_path))
hdu.writeto(output_path, overwrite=True)
print('')
#Upload the master dark to PINES server.
if upload:
print('Beginning upload process to pines.bu.edu...')
print('Note, only PINES admins will be able to upload.')
print('')
sftp.chdir('/')
sftp.chdir('data/calibrations/Hot Pixel Masks')
upload_name = output_filename
# if upload_name in sftp.listdir():
# print('WARNING: This will overwrite {} in pines.bu.edu:data/calibrations/Hot Pixel Masks/'.format(upload_name))
# upload_check = input('Do you want to continue? y/n: ')
# if upload_check == 'y':
# sftp.put(output_path,upload_name)
# print('Uploaded to pines.bu.edu:data/calibrations/Hot Pixel Masks/!')
# else:
# print('Skipping upload!')
# else:
sftp.put(output_path, upload_name)
print(
'Uploaded {} to pines.bu.edu:data/calibrations/Hot Pixel Masks/!'.
format(upload_name))
def dead_pixels(date, band, clip_lvl=4.5, box_l=3, upload=False, sftp=''):
pines_path = pines_dir_check()
flat_path = pines_path / ('Calibrations/Flats/Domeflats/' + band +
'/Master Flats/')
flat_file = natsort.natsorted(list(flat_path.rglob('*' + date +
'.fits')))[0]
master_flat = fits.open(flat_file)[0].data
shape = np.shape(master_flat)
# #Can plot a histogram of master_flat pixel values.
# his = histogram(master_flat, bins=250)
# plt.figure(figsize=(12,5))
# plt.bar(his[1][0:-1], his[0], width=0.005)
# plt.yscale('log')
# plt.xlabel('Pixel Value', fontsize=14)
# plt.ylabel('N$_{pix}$', fontsize=14)
# plt.title('Distribution of Pixel Values in Master Flat '+band+' '+date, fontsize=16)
# pdb.set_trace()
#Incorporate bad pixel information from the Kokopelli pixel mask.
kokopelli_path = pines_path / (
'Calibrations/Kokopelli Mask/kokopelli_mask.fits')
kokopelli_mask = (
1 - fits.open(kokopelli_path)[0].data).astype('int')[0:1024, :]
master_flat[np.where(kokopelli_mask == 1)] = np.nan
print('')
print('Flagging dead pixels.')
#Find hot pixels in the master dark. This uses the 8 pixels surrounding each pixel, ignoring the pixels on the edges of the detector.
#This will iterate until no new dead pixels are found.
dead_pixel_mask = np.zeros((shape[0], shape[1]), dtype='int')
total_flagged = 0
box_ls = [13, 11, 9, 7, 5]
clip_lvls = [4.5, 4.5, 4.5, 4, 4]
iteration = 1
for i in range(len(box_ls)):
box_l = box_ls[i]
clip_lvl = clip_lvls[i]
print('Box size = {} x {}.'.format(box_l, box_l))
print('Sigma clipping level = {}.'.format(clip_lvl))
print('......')
num_flagged = 999 #Initialize
box_minus = int(box_l / 2)
box_plus = int(box_l / 2) + 1
targ_pix_ind = int(box_l**2 / 2)
while (num_flagged > 1):
num_flagged = 0
pbar = ProgressBar()
for xx in pbar(range(int(box_l / 2), shape[0] - int(box_l / 2))):
for yy in range(int(box_l / 2), shape[1] - int(box_l / 2)):
if (not np.isnan(master_flat[yy, xx])
): #Only check pixels that aren't already flagged.
box_2d = master_flat[
yy - box_minus:yy + box_plus, xx - box_minus:xx +
box_plus] #2d cutout surrounding the target pixel.
box_1d = box_2d[~np.isnan(box_2d)].ravel(
) #Unraveled box, ignoring any NaNs.
neighbor_vals = np.delete(
box_1d, targ_pix_ind
) #Remove the target pixel from the 1d array.
#neighbor_vals = sigmaclip(n, low=3.5, high=3.5)[0] #Clip away other bad pixels in the box to avoid biasing the mean/standard deviation.
if master_flat[yy, xx] < (
np.mean(neighbor_vals) -
clip_lvl * np.std(neighbor_vals)
): #Flag pixel as hot if it is more than clip_lvl sigma higher than the mean of its neighbors.
dead_pixel_mask[yy, xx] = 1
master_flat[
yy,
xx] = np.nan #Set this pixel to a NaN so that it's ignored on subsequent iterations.
num_flagged += 1
iteration += 1
total_flagged += num_flagged
print(
'Iteration {}: {} new dead pixels identified, {} dead pixels total.'
.format(iteration, num_flagged, total_flagged))
print('')
print('')
print('Found {} dead pixels.'.format(total_flagged))
output_filename = 'dpm_' + band + '_' + date + '.fits'
output_path = pines_path / ('Calibrations/Dead Pixel Masks/' +
output_filename)
hdu = fits.PrimaryHDU(dead_pixel_mask)
hdu.header['HIERARCH DATE CREATED'] = datetime.utcnow().strftime(
'%Y-%m-%d') + 'T' + datetime.utcnow().strftime('%H:%M:%S')
hdu.header['HIERARCH SIGMA CLIP LVL'] = clip_lvl
#Now save to a file on your local machine.
print('')
print('Writing the file to ' + output_filename)
#Check to see if other files of this name exist.
# if os.path.exists(output_path):
# print('')
# print('WARNING: This will overwrite {}!'.format(output_path))
# dark_check = input('Do you want to continue? y/n: ')
# if dark_check == 'y':
# hdu.writeto(output_path,overwrite=True)
# print('Wrote to {}!'.format(output_path))
# else:
# print('Not overwriting!')
# else:
hdu.writeto(output_path, overwrite=True)
print('Wrote to {}!'.format(output_path))
print('')
#Upload the master dark to PINES server.
if upload:
print('Beginning upload process to pines.bu.edu...')
print('Note, only PINES admins will be able to upload.')
print('')
sftp.chdir('/')
sftp.chdir('data/calibrations/Dead Pixel Masks')
upload_name = output_filename
# if upload_name in sftp.listdir():
# print('WARNING: This will overwrite {} in pines.bu.edu:data/calibrations/Dead Pixel Masks/'.format(upload_name))
# upload_check = input('Do you want to continue? y/n: ')
# if upload_check == 'y':
# sftp.put(output_path,upload_name)
# print('Uploaded to pines.bu.edu:data/calibrations/Dead Pixel Masks/!')
# else:
# print('Skipping upload!')
# else:
sftp.put(output_path, upload_name)
print(
'Uploaded {} to pines.bu.edu:data/calibrations/Dead Pixel Masks/!'.
format(upload_name))
def pwv(target,
directory,
P_min,
P_max,
line_of_sight='target',
RA=None,
Dec=None,
plot=False,
csv=False):
"""
Compute the precipitable water vapor (PWV) at the PTO at zenith or in direction of
``target``.
Parameters
----------
directory : str
Working directory with GOES-R files in it
P_min : float
Lower pressure level (lower altitude). Range between 1100 and 0.05 (hPa)
P_max : float
Upper pressure level (higher altitude). Range between 1100 and 0.05 (hPa)
line_of_sight : str, {"target", "zenith"}
Either compute line of sight to the target or to the zenith.
RA : float
Right ascension of target (in degrees)
Dec : float
Declination of target (in degrees)
plot : bool
Generate a plot of the PWV at each time.
csv : bool
Generate a csv file of the PWV at each time.
Returns
-------
dates : list
PWV : `~numpy.ndarray`
LVT : `~numpy.ndarray`
LVM : `~numpy.ndarray`
"""
# Coordinates input in degrees
latdeg = 35.097289793027436
londeg = -111.53686622502691
site = 'Perkins Telescope Observatory'
# Access working directory
os.chdir(directory)
#nc_filesT = glob.glob('*OR_ABI-L2-LVTPF*') #MODIFIED TO WORK WITH CONUS DATA
nc_filesT = glob.glob('*OR_ABI-L2-LVTP*')
nc_filesT = sorted(nc_filesT)
#nc_filesM = glob.glob('*OR_ABI-L2-LVMPF*')
nc_filesM = glob.glob('*OR_ABI-L2-LVMP*')
nc_filesM = sorted(nc_filesM)
# Open the first file to retrieve earth parameters
Proj_info = Dataset(nc_filesT[0], 'r')
proj_info = Proj_info.variables['goes_imager_projection']
lon_origin = proj_info.longitude_of_projection_origin
H = proj_info.perspective_point_height + proj_info.semi_major_axis
r_eq = proj_info.semi_major_axis
r_pol = proj_info.semi_minor_axis
# Retrieve pressure data
P = Proj_info.variables['pressure'][:]
Proj_info.close()
Proj_info = None
# Retrieve time data
g16_data_file = []
t = []
epoch = []
date = []
day = []
hour = []
for i in range(0, len(nc_filesT)):
g16_data = nc_filesT[i]
g16_data_file.append(g16_data)
g16 = Dataset(g16_data_file[i], 'r')
ttemp = g16.variables['t'][:]
t.append(ttemp)
epochtemp = 946728000 + int(t[i])
epoch.append(epochtemp)
datetemp = time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(epoch[i]))
date.append(datetemp)
daytemp = time.strftime("%d-%m-%Y", time.gmtime(epoch[i]))
day.append(daytemp)
hourtemp = time.strftime("%H:%M:%S", time.gmtime(epoch[i]))
hour.append(hourtemp)
#Check if the output already exists, if so return.
pines_path = pines_dir_check()
short_name = short_name_creator(target)
out_date = day[1][-4:] + day[1][3:5] + day[1][0:2]
#Make the object's pwv directory if it doesn't already exist.
if not os.path.isdir(pines_path / ('Objects/' + short_name + '/pwv/')):
os.mkdir(pines_path / ('Objects/' + short_name + '/pwv/'))
output_path = pines_path / ('Objects/' + short_name +
'/pwv/PWV_los_{}.csv'.format(out_date))
if os.path.exists(output_path):
print('PWV output already exists for {}, returning.'.format(out_date))
return
# Use astropy.time to keep format for target coordinates:
times = Time(date, format='iso', scale='utc')
# Barometric formula
p0 = P[0] #hPa
R_D = 287 #Jkg-1K-1
g = 9.81 #m/s2
T_s = 288 #K
L = -0.0065 #K/m
h = (T_s / L) * ((P / p0)**(-L * R_D / g) - 1) * u.m
e = np.sqrt((r_eq**2 - r_pol**2) / (r_eq**2)) #Eccentricity
latdeg = float(latdeg)
londeg = float(londeg)
# Pressure level boundaries
P_minb = np.abs(P - P_min).argmin()
P_maxb = np.abs(P - P_max).argmin()
loc = EarthLocation(lat=latdeg * u.degree, lon=londeg * u.degree)
# Convert from radian to degrees:
raddeg = 180 / np.pi
if line_of_sight == 'zenith':
latt = latdeg
lont = londeg
elif line_of_sight == 'target':
RA = float(RA)
Dec = float(Dec)
Sky = SkyCoord(ra=RA * u.degree, dec=Dec * u.degree)
Aa = Sky.transform_to(AltAz(obstime=times, location=loc))
latt = Dec
lont = RA
# Computes PWV along line of sight
if line_of_sight == 'target':
INDEX = np.ravel(
np.where(Aa.alt.degree < 0)
) #ORIGINAL USED VALUES WERE WHERE ALT WAS >30. CHANGED FOR PERKINS.
INDEXP = np.ravel(np.where(Aa.alt.degree > 0))
# Keep time values corresponding to Alt above 30 degrees
EPOCH = epoch
for index in sorted(INDEX, reverse=True):
del EPOCH[index]
HOUR = []
for i in range(0, len(epoch)):
HOURt = time.strftime("%H:%M:%S", time.gmtime(EPOCH[i]))
HOUR.append(HOURt)
DATE = []
for i in range(0, len(epoch)):
DATEt = time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(EPOCH[i]))
DATE.append(DATEt)
Alt = Aa.alt.rad
Az = Aa.az.rad
# Compute distance from location to projection point
delta_x = []
d_lat = []
d_lon = []
for i in range(0, len(Alt)):
delta_xi = []
for j in h:
delta_xt = j / np.tan(Alt[i])
delta_xt = delta_xt * u.m**-1
delta_xi.append(delta_xt)
delta_x.append(delta_xi)
delta_x = np.array(delta_x)
for i in range(0, len(Az)):
d_latt = delta_x[i, ] * np.cos(Az[i])
d_lont = delta_x[i, ] * np.sin(Az[i])
d_lat.append(d_latt)
d_lon.append(d_lont)
d_lat = np.array(d_lat)
d_lon = np.array(d_lon)
# Compute latitude and longitude of projection points
lat_proj = []
lon_proj = []
for i in range(0, len(Alt)):
obs_latt = loc.lat.degree + raddeg * (
np.arctan(d_lat[i, ] / R_earth * u.m**1) * u.rad**-1)
obs_lont = loc.lon.degree + raddeg * (
np.arctan(d_lon[i, ] / R_earth * u.m**1) * u.rad**-1)
lat_proj.append(obs_latt)
lon_proj.append(obs_lont)
lat_proj = np.array(lat_proj)
lon_proj = np.array(lon_proj)
rad = (np.pi) / 180
lat_proj_rad = rad * lat_proj
lon_proj_rad = rad * lon_proj
lambda_0 = rad * lon_origin
#T ransform into scan angles
lat_origin = np.arctan(((r_pol**2) / (r_eq**2)) * np.tan(lat_proj_rad))
r_c = r_pol / (np.sqrt(1 - (e**2) * (np.cos(lat_origin))**2))
s_x = H - r_c * np.cos(lat_origin) * np.cos(lon_proj_rad - lambda_0)
s_y = -r_c * np.cos(lat_origin) * np.sin(lon_proj_rad - lambda_0)
s_z = r_c * np.sin(lat_origin)
s = np.sqrt(s_x**2 + s_y**2 + s_z**2)
x = np.arcsin(-s_y / s)
y = np.arctan(s_z / s_x)
g16_data_fileT = []
xscanT = []
yscanT = []
XT = []
YT = []
LVT = []
# Retrieve Temperature data
for i in INDEXP:
g16_data = nc_filesT[i]
g16_data_fileT.append(g16_data)
g16 = Dataset(nc_filesT[i], 'r')
xtemp = g16.variables['x'][:]
xscanT.append(xtemp)
ytemp = g16.variables['y'][:]
yscanT.append(ytemp)
LVTi = []
Xi = []
Yi = []
for j in range(P_minb, P_maxb + 1):
Xtemp = np.abs(xtemp - x[i, j]).argmin()
Xi.append(Xtemp)
Ytemp = np.abs(ytemp - y[i, j]).argmin()
Yi.append(Ytemp)
LVTtemp = g16.variables['LVT'][Ytemp, Xtemp, j]
LVTi.append(LVTtemp)
LVT.append(LVTi)
XT.append(Xi)
YT.append(Yi)
LVT = np.array(LVT)
# Retrieve Relative humidity data
g16_data_fileM = []
xscanM = []
yscanM = []
XM = []
YM = []
LVM = []
for i in INDEXP:
g16_dataM = nc_filesM[i]
g16_data_fileM.append(g16_dataM)
g16M = Dataset(nc_filesM[i], 'r')
xtempM = g16M.variables['x'][:]
xscanM.append(xtempM)
ytempM = g16M.variables['y'][:]
yscanM.append(ytempM)
LVMi = []
Xi = []
Yi = []
for j in range(P_minb, P_maxb + 1):
XtempM = np.abs(xtempM - x[i, j]).argmin()
Xi.append(XtempM)
YtempM = np.abs(ytempM - y[i, j]).argmin()
Yi.append(YtempM)
LVMtemp = g16M.variables['LVM'][YtempM, XtempM, j]
LVMi.append(LVMtemp)
LVM.append(LVMi)
XM.append(Xi)
YM.append(Yi)
LVM = np.array(LVM)
P = 100 * P
LVT = LVT - 273.15
Pi = P[P_minb:P_maxb + 1]
# Constants needed and integrand
rho_w = 1000 # kg/m**3
g = 9.81 #m/s**2
C = (-1) / (rho_w * g)
ev = 100 * 6.112 * LVM * np.exp(
(17.67 * LVT) /
(LVT + 243.5)) # Partial water vapour pressure in Pa
q = (0.622 * ev) / (Pi - 0.378 * ev) #Specific humdity
f = 1000 * C * q #Complete integrand multiplied by 1000 to get the PWV in mm.
# Numerical integration
PWV = []
for j in range(0, len(LVT)):
integral = 0
for i in range(1, len(Pi)):
integral = integral + (Pi[i] - Pi[i - 1]) * (
(f[j, i] + f[j, i - 1]) / 2)
PWV.append(integral)
PWV = np.asarray(PWV)
if plot:
# Plot and save data
fig = plt.figure(figsize=(15, 10))
ax = fig.add_subplot(111)
ax.plot(HOUR, PWV, 'bo', ms=4)
plt.title('PWV along line of sight, {} on {}'.format(site, day[1]),
fontsize=26)
plt.xticks(rotation='vertical', fontsize=24)
plt.yticks(fontsize=24)
ax.set_xlabel("Date", color="C0", fontsize=24)
ax.set_ylabel("PWV (mm)", color="C0", fontsize=24)
RA_patch = mpatches.Patch(color='white',
label='RA: {} degrees'.format(RA))
Dec_patch = mpatches.Patch(color='white',
label='Dec: {} degrees'.format(Dec))
every_nth = 6
for n, label in enumerate(ax.xaxis.get_ticklabels()):
if n % every_nth != 0:
label.set_visible(False)
for n, label in enumerate(ax.xaxis.get_ticklines()):
if n % every_nth != 0:
label.set_visible(False)
plt.tight_layout()
plt.legend(handles=[RA_patch, Dec_patch],
loc='lower right',
fontsize=22)
plt.show()
fig.savefig('PWV_line_of_sight_{}_{}.png'.format(site, day[1]))
if csv:
np.savetxt(output_path,
np.column_stack((date, PWV)),
delimiter=',',
fmt='%s',
header='Time,PWV',
comments='')
# Computes PWV at zenith
elif line_of_sight == 'zenith':
# Transform latitude and longitude into scan angles
rad = np.pi / 180
lambda_0 = rad * lon_origin
obs_lat_rad = rad * latt
obs_lon_rad = rad * lont
lat_origin = np.arctan(((r_pol**2) / (r_eq**2)) * np.tan(obs_lat_rad))
r_c = r_pol / (np.sqrt(1 - (e**2) * (np.cos(lat_origin))**2))
s_x = H - r_c * np.cos(lat_origin) * np.cos(obs_lon_rad - lambda_0)
s_y = -r_c * np.cos(lat_origin) * np.sin(obs_lon_rad - lambda_0)
s_z = r_c * np.sin(lat_origin)
s = np.sqrt(s_x**2 + s_y**2 + s_z**2)
x = np.arcsin(-s_y / s)
y = np.arctan(s_z / s_x)
xscanT = []
yscanT = []
# Retrieve Temperature data
LVT = []
for i in range(0, len(nc_filesT)):
g16_data = nc_filesT[i]
g16_data_file.append(g16_data)
g16 = Dataset(g16_data_file[i], 'r')
xtemp = g16.variables['x'][:]
xscanT.append(xtemp)
ytemp = g16.variables['y'][:]
yscanT.append(ytemp)
XT = []
YT = []
LVTi = []
for j in range(0, len(P)):
Xtemp = np.abs(xtemp - x).argmin()
XT.append(Xtemp)
Ytemp = np.abs(ytemp - y).argmin()
YT.append(Ytemp)
LVTtemp = g16.variables['LVT'][Ytemp, Xtemp, j]
LVTi.append(LVTtemp)
LVT.append(LVTi)
LVT = np.array(LVT)
# Retrieve Relative humidity data
g16_data_fileM = []
xscanM = []
yscanM = []
LVM = []
for i in range(0, len(nc_filesM)):
g16_dataM = nc_filesM[i]
g16_data_fileM.append(g16_dataM)
g16M = Dataset(g16_data_fileM[i], 'r')
xtempM = g16M.variables['x'][:]
xscanM.append(xtempM)
ytempM = g16M.variables['y'][:]
yscanM.append(ytempM)
XM = []
YM = []
LVMi = []
for j in range(0, len(P)):
XtempM = np.abs(xtempM - x).argmin()
XM.append(XtempM)
YtempM = np.abs(ytempM - y).argmin()
YM.append(YtempM)
LVMtemp = g16M.variables['LVM'][YtempM, XtempM, j]
LVMi.append(LVMtemp)
LVM.append(LVMi)
LVM = np.array(LVM)
# Change pressure units to Pa and Temperature to K
P = 100 * P
LVT = LVT - 273.15
# Constants needed and integrand
rho_w = 1000 # kg/m**3
g = 9.81 #m/s**2
C = (-1) / (rho_w * g)
#ev = 100*6.11*LVM*10**((7.5*LVT)/(LVT+237.15)) # Partial water vapour pressure in Pa
ev = 100 * 6.112 * LVM * np.exp((17.67 * LVT) / (LVT + 243.5))
q = (0.622 * ev) / (P - 0.378 * ev) #Specific humdity
f = 1000 * C * q #Complete integrand multiplied by 1000 to get the PWV in mm.
# Numerical integration
PWV = []
for j in range(0, len(nc_filesT)):
integral = 0
for i in range(P_minb + 1, P_maxb + 1):
integral = integral + (P[i] - P[i - 1]) * (
(f[j, i] + f[j, i - 1]) / 2)
PWV.append(integral)
PWV = np.asarray(PWV)
out_date = date
if plot:
# Plot and save data
fig = plt.figure(figsize=(15, 10))
ax = fig.add_subplot(111)
ax.plot(hour, PWV, 'bo', ms=4)
plt.title('Precipitable Water Vapor at zenith, {} on {}'.format(
site, day[1]),
fontsize=26)
plt.xticks(rotation='vertical', fontsize=24)
plt.yticks(fontsize=24)
ax.set_xlabel("Date", color="C0", fontsize=24)
ax.set_ylabel("PWV (mm)", color="C0", fontsize=24)
every_nth = 6
for n, label in enumerate(ax.xaxis.get_ticklabels()):
if n % every_nth != 0:
label.set_visible(False)
for n, label in enumerate(ax.xaxis.get_ticklines()):
if n % every_nth != 0:
label.set_visible(False)
plt.tight_layout()
plt.show()
fig.savefig('PWV_at_zenith_{}_{}.png'.format(site, day[1]))
if csv:
np.savetxt('PWV_zenith_{}_{}.csv'.format(site, day[1]),
np.column_stack((date, PWV)),
delimiter=',',
fmt='%s',
header='Time,PWV',
comments='')
return
def variable_pixels(date, exptime, clip_lvl=5, upload=False, sftp=''):
pines_path = pines_dir_check()
dark_stddev_path = pines_path / ('Calibrations/Darks/Master Darks Stddev/')
all_dark_stddev_files = natsort.natsorted(
list(Path(dark_stddev_path).rglob('*' + date + '.fits')))
dark_stddev_file = ''
for file in all_dark_stddev_files:
if float(file.name.split('_')[3]) == exptime:
dark_stddev_file = file
if dark_stddev_file == '':
raise RuntimeError('No dark stddev files found on disk with date ' +
date + ' and exposure time ' + str(exptime) +
' seconds!')
master_dark_stddev = fits.open(dark_stddev_file)[0].data
shape = np.shape(master_dark_stddev)
#Flag variable pixels, those that vary >= clip_lvl * the mean variation. Save a boolean mask of variable pixels to incorporate into the bad pixel mask.
variable_inds = np.where(
master_dark_stddev >= clip_lvl * np.nanmean(master_dark_stddev))
variable_mask = np.zeros((shape[0], shape[1]), dtype='int')
variable_mask[variable_inds] = 1
print('')
print('Found {} variable pixels.'.format(len(variable_inds[0])))
output_filename = 'vpm_' + str(exptime) + '_s_' + date + '.fits'
output_path = pines_path / ('Calibrations/Variable Pixel Masks/' +
output_filename)
#Add some header keywords detailing the master_dark creation process.
hdu = fits.PrimaryHDU(variable_mask)
hdu.header['HIERARCH DATE CREATED'] = datetime.utcnow().strftime(
'%Y-%m-%d') + 'T' + datetime.utcnow().strftime('%H:%M:%S')
hdu.header['HIERARCH SIGMA CLIP LVL'] = clip_lvl
hdu.header['HIERARCH CLIP VALUE'] = clip_lvl * np.nanmean(
master_dark_stddev)
#Now save to a file on your local machine.
print('')
# #Check to see if other files of this name exist.
# if os.path.exists(output_path):
# print('')
# print('WARNING: This will overwrite {}!'.format(output_path))
# dark_check = input('Do you want to continue? y/n: ')
# if dark_check == 'y':
# hdu.writeto(output_path,overwrite=True)
# print('Wrote to {}!'.format(output_path))
# else:
# print('Not overwriting!')
# else:
hdu.writeto(output_path, overwrite=True)
print('Wrote to {}!'.format(output_path))
#Upload the master dark to PINES server.
if upload:
print('Uploading to pines.bu.edu...')
sftp.chdir('/')
sftp.chdir('data/calibrations/Variable Pixel Masks')
upload_name = output_filename
# if upload_name in sftp.listdir():
# print('WARNING: This will overwrite {} in pines.bu.edu:data/calibrations/Variable Pixel Masks/'.format(upload_name))
# upload_check = input('Do you want to continue? y/n: ')
# if upload_check == 'y':
# sftp.put(output_path,upload_name)
# print('Uploaded to pines.bu.edu:data/calibrations/Variable Pixel Masks/!')
# else:
# print('Skipping upload!')
# else:
sftp.put(output_path, upload_name)
print(
'Uploaded {} to pines.bu.edu:data/calibrations/Variable Pixel Masks/!'
.format(upload_name))