def get_master_synthetic_image(sftp, target_name):
pines_path = pines_dir_check()
sftp.chdir('/data/master_images/')
synthetic_filename = target_name.replace(' ',
'') + '_master_synthetic.fits'
download_path = pines_path / ('Calibrations/Master Synthetic Images/' +
synthetic_filename)
sftp.get(synthetic_filename, download_path)
print('Downloaded {} to {}!'.format(synthetic_filename, download_path))
def bad_shift_identifier(target, date, bad_shift_threshold=200):
pines_path = pines_dir_check()
log_path = pines_path / ('Logs/' + date + '_log.txt')
log = pines_log_reader(log_path)
target_inds = np.where(log['Target'] == target)[0]
x_shifts = np.array(log['X shift'][target_inds])
y_shifts = np.array(log['Y shift'][target_inds])
bad_shift_inds = np.where((x_shifts > bad_shift_threshold)
| (y_shifts > bad_shift_threshold))[0]
shift_flags = np.zeros(len(target_inds), dtype=int)
shift_flags[bad_shift_inds] = 1
return
def straggler_upload(sftp, targets, delete_raw=False):
sftp.chdir('/data/reduced/mimir/')
runs = sftp.listdir()
pines_path = pines_dir_check()
for i in range(len(targets)):
target = targets[i]
short_name = short_name_creator(target)
raw_path = pines_path / ('Objects/' + short_name + '/raw/')
red_path = pines_path / ('Objects/' + short_name + '/reduced/')
raw_files = np.array(natsorted(glob(str(raw_path) + '/*.fits')))
for j in range(len(raw_files)):
sftp.chdir('/data/reduced/mimir/')
red_file = str(red_path) + '/' + raw_files[j].split('/')[-1].split(
'.fits')[0] + '_red.fits'
night = red_file.split('/')[-1].split('.')[0]
run_guess = night[0:6]
ind = np.where(np.array(runs) == run_guess)[0][0]
if ind + 1 == len(runs):
inds = np.arange(ind - 1, ind + 1)
runs = np.array(runs)[inds]
runs = [runs[1], runs[0]]
else:
inds = np.arange(ind - 1, ind + 2)
runs = np.array(runs)[inds]
runs = [runs[1], runs[0], runs[2]]
found = False
for k in range(len(runs)):
sftp.chdir(runs[k])
if night in sftp.listdir():
print('Uploading {} to {}.'.format(
red_file.split('/')[-1], sftp.pwd + '/' + night))
sftp.chdir(night)
sftp.put(red_file, red_file.split('/')[-1])
found = True
else:
sftp.chdir('..')
if found:
break
pdb.set_trace()
def output_wrangler(target):
pines_path = pines_dir_check()
short_name = short_name_creator(target)
object_path = pines_path / ('Objects/' + short_name)
sources_path = object_path / ('sources')
aper_phot_path = object_path / ('aper_phot')
analysis_path = object_path / ('analysis')
output_path = object_path / ('output')
#Copy souce detect image.
src = sources_path / ('target_and_refs.png')
dest = output_path / ('target_and_refs.png')
shutil.copyfile(src, dest)
#Copy image centroids.
src = sources_path / ('target_and_references_centroids.csv')
dest = output_path / ('target_and_references_centroids.csv')
shutil.copyfile(src, dest)
#Copy best aperture raw photometry.
best_ap_path = analysis_path / ('optimal_aperture.txt')
with open(best_ap_path, 'r') as f:
best_ap = f.readlines()[0]
src = aper_phot_path / (short_name + '_aper_phot_' + best_ap +
'_pix_radius.csv')
dest = output_path / (short_name + '_aper_phot_' + best_ap +
'_pix_radius.csv')
shutil.copyfile(src, dest)
#Copy best aperture weighted lightcurve photometry.
src = analysis_path / ('aper_phot_analysis/' + best_ap + '/' + short_name +
'_weighted_lc_aper_phot_' + best_ap + '_pix.csv')
dest = output_path / (short_name + '_weighted_lc_aper_phot_' + best_ap +
'_pix.csv')
shutil.copyfile(src, dest)
pdb.set_trace()
return
def align_and_stack_images(target):
pines_path = pines_dir_check()
short_name = short_name_creator(target)
reduced_path = pines_path / ('Objects/' + short_name + '/reduced')
log_path = pines_path / 'Logs/'
reduced_files = np.array(
natsorted([x for x in reduced_path.glob('*red.fits')]))
dates = np.array(list(set([i.name.split('.')[0] for i in reduced_files])))
log_names = np.array(natsorted([i + '_log.txt' for i in dates]))
aligned_files = []
for i in range(len(log_names)):
log = log_path / log_names[i]
log_df = pat.utils.pines_log_reader(log)
locs = np.where((abs(log_df['X shift']) < 0.5)
& (abs(log_df['Y shift']) < 0.5))[0]
good_files = np.array(
log_df['Filename'][np.where((abs(log_df['X shift']) < 0.5)
& (abs(log_df['Y shift']) < 0.5))[0]])
good_files = [i.split('.fits')[0] + '_red.fits' for i in good_files]
aligned_files.extend(good_files)
pdb.set_trace()
return
def absolute_image_position_plot(target, centroided_sources):
pines_path = pines_dir_check()
short_name = short_name_creator(target)
#Get plot style parameters.
title_size, axis_title_size, axis_ticks_font_size, legend_font_size = plot_style(
)
#Get list of souce names in the centroid output.
source_names = get_source_names(centroided_sources)
centroided_sources.columns = centroided_sources.keys().str.strip()
#Get times from the centroid output and split them by night.
times_full = np.array(centroided_sources['Time (JD UTC)'])
night_inds = night_splitter(times_full)
num_nights = len(night_inds)
times_nights = [times_full[night_inds[i]] for i in range(num_nights)]
standard_x = standard_x_range(times_nights)
source = source_names[0]
fig, ax = plt.subplots(nrows=2,
ncols=num_nights,
figsize=(17, 9),
sharex='col',
sharey='row')
plt.subplots_adjust(left=0.07,
hspace=0.05,
wspace=0.05,
top=0.92,
bottom=0.17)
for j in range(num_nights):
if j == 0:
ax[0, j].set_ylabel('Image X', fontsize=axis_title_size)
ax[1, j].set_ylabel('Image Y', fontsize=axis_title_size)
inds = night_inds[j]
times = times_nights[j]
absolute_x = np.array(centroided_sources[source + ' Image X'][inds],
dtype='float')
absolute_y = np.array(centroided_sources[source + ' Image Y'][inds],
dtype='float')
ax[0, j].plot(times,
absolute_x,
marker='.',
linestyle='',
alpha=0.3,
color='tab:blue',
label='Raw x')
ax[1, j].plot(times,
absolute_y,
marker='.',
linestyle='',
alpha=0.3,
color='tab:orange',
label='Raw y')
#bin
block_inds = block_splitter(times)
block_times = np.zeros(len(block_inds))
block_x = np.zeros(len(block_inds))
block_x_err = np.zeros(len(block_inds))
block_y = np.zeros(len(block_inds))
block_y_err = np.zeros(len(block_inds))
for k in range(len(block_inds)):
try:
block_times[k] = np.nanmean(times[block_inds[k]])
block_x[k] = np.nanmean(absolute_x[block_inds[k]])
block_x_err[k] = np.nanstd(
absolute_x[block_inds[k]]) / np.sqrt(
len(absolute_x[block_inds[k]]))
block_y[k] = np.nanmean(absolute_y[block_inds[k]])
block_y_err[k] = np.nanstd(
absolute_y[block_inds[k]]) / np.sqrt(
len(absolute_y[block_inds[k]]))
except:
pdb.set_trace()
ax[0, j].errorbar(block_times,
block_x,
block_x_err,
marker='o',
linestyle='',
color='tab:blue',
ms=8,
mfc='none',
mew=2,
label='Bin x')
ax[1, j].errorbar(block_times,
block_y,
block_y_err,
marker='o',
linestyle='',
color='tab:orange',
ms=8,
mfc='none',
mew=2,
label='Bin y')
ax[0, j].tick_params(labelsize=axis_ticks_font_size)
ax[1, j].tick_params(labelsize=axis_ticks_font_size)
ax[0, j].grid(alpha=0.2)
ax[1, j].grid(alpha=0.2)
ax[1, j].set_xlabel('Time (JD UTC)', fontsize=axis_title_size)
if j == num_nights - 1:
ax[0, j].legend(bbox_to_anchor=(1.29, 1),
fontsize=legend_font_size)
ax[1, j].legend(bbox_to_anchor=(1.29, 1),
fontsize=legend_font_size)
plt.suptitle(source + ' Image Centroid Positions', fontsize=title_size)
#plt.subplots_adjust(left=0.07, hspace=0.05, wspace=0.05, top=0.92, bottom=0.08, right=0.85)
#ax.legend(bbox_to_anchor=(1.01, 1), fontsize=14)
ax[0, j].set_xlim(
np.mean(times) - standard_x / 2,
np.mean(times) + standard_x / 2)
ax[1, j].set_xlim(
np.mean(times) - standard_x / 2,
np.mean(times) + standard_x / 2)
output_filename = pines_path / ('Objects/' + short_name +
'/analysis/diagnostic_plots/' + source +
'_image_positions.png')
plt.savefig(output_filename, dpi=300)
return
def master_synthetic_image_creator(target,
image_name,
seeing=2.5,
sigma_above_bg=5):
'''Authors:
Patrick Tamburo, Boston University, February 2021
Purpose:
Creates a master synthetic image for a PINES target by detecting sources in a reduced image of the field.
Inputs:
target (str): The target's full 2MASS name.
image_name (str): The name of the reduced file (e.g., 20210204.420_red.fits).
seeing (float): The FWHM seeing of the image in arcsec. By default, 2.5".
sigma_above_bg (float): The sigma above background used to rule in sources in daostarfinder. By default, 5.
Outputs:
Writes out a master synthetic image to PINES_analysis_toolkit/Calibrations/Master Synthetic Images/.
TODO:
Write the filename used to create the master image to the master image header.
'''
def mimir_source_finder(image_path,
sigma_above_bg,
fwhm,
exclude_lower_left=False):
"""Find sources in Mimir images."""
np.seterr(all='ignore') #Ignore invalids (i.e. divide by zeros)
#Find stars in the master image.
avg, med, stddev = sigma_clipped_stats(
image, sigma=3.0, maxiters=3) #Previously maxiters = 5!
daofind = DAOStarFinder(fwhm=fwhm,
threshold=sigma_above_bg * stddev,
sky=med,
ratio=0.8)
new_sources = daofind(image)
x_centroids = new_sources['xcentroid']
y_centroids = new_sources['ycentroid']
sharpness = new_sources['sharpness']
fluxes = new_sources['flux']
peaks = new_sources['peak']
#Cut sources that are found within 20 pix of the edges.
use_x = np.where((x_centroids > 20) & (x_centroids < 1004))[0]
x_centroids = x_centroids[use_x]
y_centroids = y_centroids[use_x]
sharpness = sharpness[use_x]
fluxes = fluxes[use_x]
peaks = peaks[use_x]
use_y = np.where((y_centroids > 20) & (y_centroids < 1004))[0]
x_centroids = x_centroids[use_y]
y_centroids = y_centroids[use_y]
sharpness = sharpness[use_y]
fluxes = fluxes[use_y]
peaks = peaks[use_y]
#Also cut using sharpness, this seems to eliminate a lot of false detections.
use_sharp = np.where(sharpness > 0.5)[0]
x_centroids = x_centroids[use_sharp]
y_centroids = y_centroids[use_sharp]
sharpness = sharpness[use_sharp]
fluxes = fluxes[use_sharp]
peaks = peaks[use_sharp]
if exclude_lower_left:
#Cut sources in the lower left, if bars are present.
use_ll = np.where((x_centroids > 512) | (y_centroids > 512))
x_centroids = x_centroids[use_ll]
y_centroids = y_centroids[use_ll]
sharpness = sharpness[use_ll]
fluxes = fluxes[use_ll]
peaks = peaks[use_ll]
#Cut targets whose y centroids are near y = 512. These are usually bad.
use_512 = np.where(
np.logical_or((y_centroids < 510), (y_centroids > 514)))[0]
x_centroids = x_centroids[use_512]
y_centroids = y_centroids[use_512]
sharpness = sharpness[use_512]
fluxes = fluxes[use_512]
peaks = peaks[use_512]
#Cut sources with negative/saturated peaks
use_peaks = np.where((peaks > 30) & (peaks < 3000))[0]
x_centroids = x_centroids[use_peaks]
y_centroids = y_centroids[use_peaks]
sharpness = sharpness[use_peaks]
fluxes = fluxes[use_peaks]
peaks = peaks[use_peaks]
#Do quick photometry on the remaining sources.
positions = [(x_centroids[i], y_centroids[i])
for i in range(len(x_centroids))]
apertures = CircularAperture(positions, r=4)
phot_table = aperture_photometry(image - med, apertures)
#Cut based on brightness.
phot_table.sort('aperture_sum')
cutoff = 1 * std * np.pi * 4**2
bad_source_locs = np.where(phot_table['aperture_sum'] < cutoff)
phot_table.remove_rows(bad_source_locs)
x_centroids = phot_table['xcenter'].value
y_centroids = phot_table['ycenter'].value
return (x_centroids, y_centroids)
def synthetic_image_maker(x_centroids, y_centroids, fwhm):
#Construct synthetic images from centroid/flux data.
synthetic_image = np.zeros((1024, 1024))
sigma = fwhm / 2.355
for i in range(len(x_centroids)):
#Cut out little boxes around each source and add in Gaussian representations. This saves time.
int_centroid_x = int(np.round(x_centroids[i]))
int_centroid_y = int(np.round(y_centroids[i]))
y_cut, x_cut = np.mgrid[int_centroid_y - 10:int_centroid_y + 10,
int_centroid_x - 10:int_centroid_x + 10]
dist = np.sqrt((x_cut - x_centroids[i])**2 +
(y_cut - y_centroids[i])**2)
synthetic_image[y_cut,
x_cut] += np.exp(-((dist)**2 / (2 * sigma**2) +
((dist)**2 / (2 * sigma**2))))
return (synthetic_image)
target = target.replace(' ', '')
pines_path = pines_dir_check()
short_name = short_name_creator(target)
master_synthetic_path = pines_path / (
'Calibrations/Master Synthetic Images/' + target +
'_master_synthetic.fits')
image_path = pines_path / ('Objects/' + short_name + '/reduced/' +
image_name)
plt.ion()
target = target.replace(' ', '')
seeing = float(seeing)
daostarfinder_fwhm = seeing * 2.355 / 0.579
#Open the image and calibration files.
header = fits.open(image_path)[0].header
image = fits.open(image_path)[0].data
#Interpolate over bad pixels
kernel = Gaussian2DKernel(x_stddev=1)
image = interpolate_replace_nans(image, kernel)
#Do a simple 2d background model.
box_size = 32
sigma_clip = SigmaClip(sigma=3.)
bkg_estimator = MedianBackground()
bkg = Background2D(image, (box_size, box_size),
filter_size=(3, 3),
sigma_clip=sigma_clip,
bkg_estimator=bkg_estimator)
image = image - bkg.background
avg, med, std = sigma_clipped_stats(image)
#Find sources in the image.
(x_centroids,
y_centroids) = mimir_source_finder(image,
sigma_above_bg=sigma_above_bg,
fwhm=daostarfinder_fwhm)
#Plot the field with detected sources.
qp(image)
plt.plot(x_centroids, y_centroids, 'rx')
for i in range(len(x_centroids)):
plt.text(x_centroids[i] + 8,
y_centroids[i] + 8,
str(i),
color='r',
fontsize=14)
plt.title(
'Inspect to make sure stars were found!\nO for magnification tool, R to reset view'
)
plt.tight_layout()
plt.show()
print('')
print('')
print('')
#Prompt the user to remove any false detections.
ids = input(
'Enter ids of sources to be removed separated by commas (i.e., 4,18,22). If none to remove, hit enter. To break, ctrl + D. '
)
if ids != '':
ids_to_eliminate = [int(i) for i in ids.split(',')]
ids = [
int(i) for i in np.linspace(0,
len(x_centroids) - 1, len(x_centroids))
]
ids_to_keep = []
for i in range(len(ids)):
if ids[i] not in ids_to_eliminate:
ids_to_keep.append(ids[i])
else:
ids_to_keep = [
int(i) for i in np.linspace(0,
len(x_centroids) - 1, len(x_centroids))
]
plt.clf()
plt.imshow(image, origin='lower', vmin=med, vmax=med + 5 * std)
plt.plot(x_centroids[ids_to_keep], y_centroids[ids_to_keep], 'rx')
for i in range(len(x_centroids[ids_to_keep])):
plt.text(x_centroids[ids_to_keep][i] + 8,
y_centroids[ids_to_keep][i] + 8,
str(i),
color='r')
#Create the synthetic image using the accepted sources.
synthetic_image = synthetic_image_maker(x_centroids[ids_to_keep],
y_centroids[ids_to_keep], 8)
plt.figure(figsize=(9, 7))
plt.imshow(synthetic_image, origin='lower')
plt.title('Synthetic image')
plt.show()
pdb.set_trace()
print('')
print('')
print('')
#Now write to a master synthetic image.fits file.
hdu = fits.PrimaryHDU(synthetic_image)
hdu.writeto(master_synthetic_path, overwrite=True)
print(
'Writing master synthetic image to {}/'.format(master_synthetic_path))
def corr_all_sources_plot(target):
print('Generating corrected flux plots for all sources...\n')
pines_path = pines_dir_check()
short_name = short_name_creator(target)
analysis_path = pines_path / ('Objects/' + short_name + '/analysis/')
photometry_path = pines_path / ('Objects/' + short_name + '/aper_phot/')
#Grab the data for the best aperture.
if os.path.exists(analysis_path / ('optimal_aperture.txt')):
with open(analysis_path / ('optimal_aperture.txt'), 'r') as f:
best_ap = f.readlines()[0].split(': ')[1].split('\n')[0]
phot_type = best_ap.split('_')[1]
if phot_type == 'fixed':
s = 'r'
elif phot_type == 'variable':
s = 'f'
else:
raise RuntimeError(
'No optimal_aperture.txt file for {}.\nUsing first photometry file in {}.'
.format(target, phot_path))
filename = short_name.replace(
' ', '') + '_' + phot_type + '_aper_phot_' + s + '=' + best_ap.split(
'_')[0] + '_nightly_weighted_lc.csv'
best_phot_path = analysis_path / ('aper_phot_analysis/' + best_ap + '/')
output_path = best_phot_path / ('corr_ref_plots/')
if not os.path.exists(output_path):
os.mkdir(output_path)
data = pines_log_reader(best_phot_path / filename)
ref_names = get_source_names(data)[1:]
num_refs = len(ref_names)
times = np.array(data['Time BJD TDB'])
night_inds = night_splitter(times)
num_nights = len(night_inds)
cmap = plt.get_cmap('viridis')
for i in range(num_refs + 1):
fig, ax = plt.subplots(nrows=1,
ncols=num_nights,
figsize=(17, 5),
sharey=True)
plt.subplots_adjust(left=0.07, wspace=0.05, top=0.92, bottom=0.17)
if i == 0:
color = cmap(0)
flux = np.array(data[short_name + ' Corrected Flux'],
dtype='float64')
flux_err = np.array(data[short_name + ' Corrected Flux Error'],
dtype='float64')
title = short_name
output_name = short_name + '_corrected_flux.png'
else:
color = cmap(95)
ref_name = ref_names[i - 1]
flux = np.array(data[ref_name + ' Corrected Flux'],
dtype='float64')
flux_err = np.array(data[ref_name + ' Corrected Flux Error'],
dtype='float64')
if i < 10:
num = '0' + str(i)
else:
num = str(i)
output_name = 'reference_' + num + '_corrected_flux.png'
for j in range(num_nights):
if i != 0:
weight = np.array(data[ref_name +
' ALC Weight'])[night_inds[j]][0]
title = ref_name.replace(
'erence', '.') + ', weight = {:1.3f}'.format(weight)
if j == 0:
ax[j].set_ylabel('Normalized Flux', fontsize=20)
inds = night_inds[j]
block_inds = block_splitter(times[inds])
binned_time = []
binned_flux = []
binned_err = []
for k in range(len(block_inds)):
binned_time.append(np.nanmean(times[inds][block_inds[k]]))
binned_flux.append(np.nanmean(flux[inds][block_inds[k]]))
binned_err.append(
np.nanstd(flux[inds][block_inds[k]]) /
np.sqrt(len(block_inds[k])))
ax[j].plot(times[inds],
flux[inds],
color=color,
linestyle='',
marker='.',
alpha=0.25)
ax[j].errorbar(binned_time,
binned_flux,
binned_err,
color=color,
linestyle='',
marker='o',
ms=10,
mfc='none',
mew=2)
ax[j].set_xlabel('Time (BJD$_{TDB}$)', fontsize=20)
ax[j].tick_params(labelsize=16)
ax[j].axhline(1, color='k', alpha=0.7, lw=1, zorder=0)
ax[j].grid(alpha=0.2)
ax[j].set_title(title, fontsize=20, color=color)
ax[j].set_ylim(0.9, 1.1)
plt.savefig(output_path / output_name, dpi=300)
plt.close()
seeing = np.array(seeing[np.where(~np.isnan(seeing))], dtype=float)
seeing = seeing[np.where((seeing > 1.2) & (seeing < 7.0))[0]]
mean_seeing = np.nanmean(seeing)
std_seeing = np.nanstd(seeing)
print('Average seeing for {}: {:1.1f} +/- {:1.1f}"'.format(
log_path.split('/')[-1].split('_')[0], mean_seeing,
std_seeing))
return mean_seeing
except:
print('{}: No seeing measurements, inspect manually.'.format(
log_path.split('/')[-1].split('_')[0]))
return np.nan
if __name__ == '__main__':
pines_dir = pines_dir_check()
log_path = pines_dir / 'Logs'
logs = np.array(natsorted(glob(str(log_path / '*.txt'))))
#Remove one bad log
bad_loc = np.where(
np.array([logs[i].split('/')[-1]
for i in range(len(logs))]) == '20201003_log.txt')[0][0]
logs = np.delete(logs, bad_loc)
#logs = logs[6:-1] #Ignore first set of logs, ignore master_log
seeings = np.zeros(len(logs))
for i in range(len(logs)):
log_path = logs[i]
seeings[i] = average_seeing(log_path)
def get_reduced_science_files(sftp, target_name):
t1 = time.time()
#Get the user's pines_analysis_toolkit path
pines_path = pines_dir_check()
#Get the target's short name and set up a data directory, if necessary.
short_name = short_name_creator(target_name)
if not os.path.exists(pines_path / ('Objects/' + short_name)):
object_directory_creator(pines_path, short_name)
reduced_data_path = pines_path / ('Objects/' + short_name + '/reduced/')
dark_path = pines_path / ('Calibrations/Darks')
flats_path = pines_path / ('Calibrations/Flats/Domeflats')
#Grab an up-to-date copy of the master log, which will be used to find images.
get_master_log(sftp, pines_path)
#Let's grab all of the available calibration data on pines.bu.edu.
get_calibrations(sftp, pines_path)
print('Calibrations up to date!')
time.sleep(2)
#Read in the master target list and find images of the requested target.
df = pines_log_reader(pines_path / ('Logs/master_log.txt'))
targ_inds = np.where(np.array(df['Target']) == target_name)[0]
file_names = np.array(df['Filename'])[targ_inds]
print('')
print('Searching pines.bu.edu for reduced science files for {}.'.format(
target_name))
print('')
#Get list of dates that data are from, in chronological order.
dates = [
int(i) for i in list(
set([
str.split(file_names[i], '.')[0]
for i in range(len(file_names))
]))
]
dates = np.array(dates)[np.argsort(dates)]
comp_dates = np.array(
[int(file_names[i].split('.')[0]) for i in range(len(file_names))])
print('Found ', len(file_names), ' raw files for ', target_name, ' on ',
len(dates), ' dates.')
date_holder = [[] for x in range(len(dates))]
for i in range(len(dates)):
date = dates[i]
print(date, ': ', len(np.where(comp_dates == date)[0]), ' files.')
date_holder[i].extend(file_names[np.where(comp_dates == date)[0]])
time.sleep(0.1)
dates = [str(i) for i in dates]
#Now download the identified data.
sftp.chdir('/data/reduced/mimir')
run_dirs = sftp.listdir()
file_num = 1
for i in range(len(run_dirs)):
sftp.chdir(run_dirs[i])
night_dirs = sftp.listdir()
for j in range(len(night_dirs)):
night_check = night_dirs[j]
if night_check in dates:
sftp.chdir(night_check)
date_holder_ind = np.where(
np.array(dates) == night_check)[0][0]
files = date_holder[date_holder_ind]
files_in_path = sftp.listdir()
for k in range(len(files)):
download_filename = files[k].split(
'.fits')[0] + '_red.fits'
if not (reduced_data_path / download_filename).exists():
if download_filename in files_in_path:
print('Downloading to {}, {} of {}'.format(
reduced_data_path / download_filename,
file_num, len(file_names)))
sftp.get(download_filename,
reduced_data_path / download_filename)
else:
print(
'A reduced image does not yet exist for {}, ask an administrator to make one!'
.format(files[k]))
else:
print('{} already in {}, skipping.'.format(
download_filename, reduced_data_path))
file_num += 1
sftp.chdir('..')
sftp.chdir('..')
print('')
#Now grab the logs.
sftp.chdir('/data/logs')
for i in range(len(dates)):
log_name = dates[i] + '_log.txt'
print('Downloading {} to {}.'.format(log_name, pines_path /
('Logs/' + log_name)))
sftp.get(log_name, pines_path / ('Logs/' + log_name))
# sftp.chdir('/data/raw/mimir')
# print('')
# for i in range(len(run_dirs)):
# sftp.chdir(run_dirs[i])
# night_dirs = sftp.listdir()
# for j in range(len(night_dirs)):
# night_check = night_dirs[j]
# if night_check in dates:
# sftp.chdir(night_check)
# log_name = night_check+'_log.txt'
# files_in_path = sftp.listdir()
# if log_name in files_in_path:
# if not (pines_path/('Logs/'+log_name)).exists():
# sftp.get(log_name,pines_path/('Logs/'+log_name))
# print('Downloading {} to {}.'.format(log_name, pines_path/('Logs/'+log_name)))
# else:
# print('{} already in {}, skipping.'.format(log_name,pines_path/'Logs/'))
# sftp.chdir('..')
# sftp.chdir('..')
print('')
print('get_reduced_science_files runtime: ',
np.round((time.time() - t1) / 60, 1), ' minutes.')
print('Done!')
def dome_flat_field(date,
band,
lights_on_start=0,
lights_on_stop=0,
lights_off_start=0,
lights_off_stop=0,
upload=False,
delete_raw=False,
sftp=''):
clip_lvl = 3 #The value to use for sigma clipping.
np.seterr(
invalid='ignore'
) #Suppress some warnings we don't care about in median combining.
plt.ion() #Turn on interactive plotting.
pines_path = pines_dir_check()
t1 = time.time()
#If an sftp connection to the PINES server was passed, download the flat data.
if type(sftp) == pysftp.Connection:
sftp.chdir('/')
sftp.chdir('data/raw/mimir')
run_list = sftp.listdir()
data_path = '' #Initialize to check that it gets filled.
for i in range(len(run_list)):
sftp.chdir(run_list[i])
date_list = sftp.listdir()
if date in date_list:
data_path = sftp.getcwd()
print('{} directory found in pines.bu.edu:{}/'.format(
date, data_path))
print('')
sftp.chdir(date)
break
sftp.chdir('..')
if data_path == '':
print(
'ERROR: {} not found in any run on pines.bu.edu:data/raw/mimir/.'
.format(date))
return
else:
#If the file start/stop numbers are specfied, grab those files.
if (lights_on_stop != 0) or (lights_off_stop != 0):
files_in_dir = sftp.listdir()
on_flat_filenums = np.arange(lights_on_start,
lights_on_stop + 1,
step=1)
off_flat_filenums = np.arange(lights_off_start,
lights_off_stop + 1,
step=1)
flat_files = []
lights_on_files = []
lights_off_files = []
#Add the lights-on flats to the file list.
for i in range(len(on_flat_filenums)):
file_num = on_flat_filenums[i]
#Generate the filename.
if file_num < 10:
file_name = date + '.00' + str(file_num) + '.fits'
elif (file_num >= 10) and (file_num < 100):
file_name = date + '.0' + str(file_num) + '.fits'
else:
file_name = date + '.' + str(file_num) + '.fits'
#Check if the file name is in the directory, and if so, append it to the list of flat files.
if file_name in files_in_dir:
flat_files.append(file_name)
lights_on_files.append(file_name)
else:
print('{} not found in directory, skipping.'.format(
file_name))
#Do the same for the lights-off files.
for i in range(len(off_flat_filenums)):
file_num = off_flat_filenums[i]
#Generate the filename.
if file_num < 10:
file_name = date + '.00' + str(file_num) + '.fits'
elif (file_num >= 10) and (file_num < 100):
file_name = date + '.0' + str(file_num) + '.fits'
else:
file_name = date + '.' + str(file_num) + '.fits'
#Check if the file name is in the directory, and if so, append it to the list of flat files.
if file_name in files_in_dir:
flat_files.append(file_name)
lights_off_files.append(file_name)
else:
print('{} not found in directory, skipping.'.format(
file_name))
#Otherwise, find the files automatically using the night's log.
else:
log_path = pines_path / 'Logs'
#Check if you already have the log for this date, if not, download it.
#Download from the /data/logs/ directory on PINES.
if not (log_path / (date + '_log.txt')).exists():
print('Downloading {}_log.txt to {}'.format(
date, log_path))
sftp.get('/data/logs/' + date + '_log.txt',
log_path / (date + '_log.txt'))
log = pines_log_reader(log_path / (date + '_log.txt'))
#Identify flat files.
flat_inds = np.where((log['Target'] == 'Flat')
& (log['Filename'] != 'test.fits')
& (log['Filt.'] == band))[0]
flat_files = natsort.natsorted(
list(set(log['Filename'][flat_inds]))
) #Set guarantees we only grab the unique files that have been identified as flats, in case the log bugged out.
print('Found {} flat files.'.format(len(flat_files)))
print('')
#Download data to the appropriate Calibrations/Flats/Domeflats/band/Raw/ directory.
dome_flat_raw_path = pines_path / (
'Calibrations/Flats/Domeflats/' + band + '/Raw')
for j in range(len(flat_files)):
if not (dome_flat_raw_path / flat_files[j]).exists():
sftp.get(flat_files[j],
(dome_flat_raw_path / flat_files[j]))
print('Downloading {} to {}, {} of {}.'.format(
flat_files[j], dome_flat_raw_path, j + 1,
len(flat_files)))
else:
print('{} already in {}, skipping download.'.format(
flat_files[j], dome_flat_raw_path))
print('')
if (lights_on_stop == 0) and (lights_off_stop == 0):
#Find the lights-on and lights-off flat files.
lights_on_files = []
lights_off_files = []
for j in range(len(flat_files)):
header = fits.open(dome_flat_raw_path /
flat_files[j])[0].header
if header['FILTNME2'] != band:
print(
'ERROR: {} taken in filter other than {}. Double check your date, try specifying start/stop file numbers, etc.'
.format(flat_files[j], band))
return
if header['OBJECT'] == 'dome_lamp_on':
lights_on_files.append(flat_files[j])
elif header['OBJECT'] == 'dome_lamp_off':
lights_off_files.append(flat_files[j])
else:
print(
"ERROR: header['OBJECT'] for {} is not 'dome_lamp_on' or 'dome_lamp_off. Double check your date, try specifying start/stop file numbers, etc."
.format(flat_files[j]))
else:
dome_flat_raw_path = pines_path / ('Calibrations/Flats/Domeflats/' +
band + '/Raw')
flat_files = natsort.natsorted(
list(Path(dome_flat_raw_path).rglob(date + '*.fits')))
#Find the lights-on and lights-off flat files.
lights_on_files = []
lights_off_files = []
for j in range(len(flat_files)):
header = fits.open(dome_flat_raw_path / flat_files[j])[0].header
if header['FILTNME2'] != band:
print(
'ERROR: {} taken in filter other than {}. Double check your date, try specifying start/stop file numbers, etc.'
.format(flat_files[j], band))
return
if header['OBJECT'] == 'dome_lamp_on':
lights_on_files.append(flat_files[j])
elif header['OBJECT'] == 'dome_lamp_off':
lights_off_files.append(flat_files[j])
else:
print(
"ERROR: header['OBJECT'] for {} is not 'dome_lamp_on' or 'dome_lamp_off. Double check your date, try specifying start/stop file numbers, etc."
.format(flat_files[j]))
if len(lights_on_files) == 0 or len(lights_off_files) == 0:
raise RuntimeError('No raw lights on/off flat files found with date ' +
date + ' in ' + band + ' band!')
print('Found {} lights-on flat files.'.format(len(lights_on_files)))
print('Found {} lights-off flat files.'.format(len(lights_off_files)))
print('')
time.sleep(1)
#Make cube of the lights-on images.
num_images = len(lights_on_files)
print('Reading in ', num_images, ' lights-on flat images.')
flat_lights_on_cube_raw = np.zeros(
[len(lights_on_files), 1024,
1024]) #Declare datatype to match raw mimir data.
print('')
print('Flat frame information')
print('-------------------------------------------------')
print('ID Mean Stddev Max Min')
print('-------------------------------------------------')
lights_on_std_devs = np.zeros(num_images)
for j in range(len(lights_on_files)):
image_data = fits.open(dome_flat_raw_path /
lights_on_files[j])[0].data[0:1024, :]
header = fits.open(dome_flat_raw_path / lights_on_files[j])[0].header
if header['FILTNME2'] != band:
print(
'ERROR: {} taken in filter other than {}. Double check your date, try specifying start/stop file numbers, etc.'
.format(lights_on_files[j], band))
return
flat_lights_on_cube_raw[
j, :, :] = image_data #This line trims off the top two rows of the image, which are overscan.
lights_on_std_devs[j] = np.std(
image_data
) #Save standard deviation of flat images to identify flats with "ski jump" horizontal bars issue.
print(
str(j + 1) + ' ' + str(np.mean(image_data)) + ' ' +
str(np.std(image_data)) + ' ' + str(np.std(image_data)) +
' ' + str(np.amin(image_data)))
#Identify bad lights-on flat images (usually have bright horizontal bands at the top/bottom of images.)
vals, lo, hi = sigmaclip(lights_on_std_devs)
bad_locs = np.where((lights_on_std_devs < lo)
| (lights_on_std_devs > hi))[0]
good_locs = np.where((lights_on_std_devs > lo)
& (lights_on_std_devs < hi))[0]
if len(bad_locs > 0):
print(
'Found {} bad lights-on flats: {}.\nExcluding from combination step.\n'
.format(len(bad_locs),
np.array(lights_on_files)[bad_locs]))
time.sleep(2)
flat_lights_on_cube_raw = flat_lights_on_cube_raw[good_locs]
#For each pixel, calculate the mean, median, and standard deviation "through the stack" of lights on flat images.
lights_on_cube_shape = np.shape(flat_lights_on_cube_raw)
master_flat_lights_on = np.zeros(
(lights_on_cube_shape[1], lights_on_cube_shape[2]), dtype='float32')
master_flat_lights_on_stddev = np.zeros(
(lights_on_cube_shape[1], lights_on_cube_shape[2]), dtype='float32')
print('')
print('Combining the lights-on flats.')
print('......')
pbar = ProgressBar()
for x in pbar(range(lights_on_cube_shape[1])):
for y in range(lights_on_cube_shape[2]):
through_stack = flat_lights_on_cube_raw[:, y, x]
through_stack_median = np.nanmedian(through_stack)
through_stack_stddev = np.nanstd(through_stack)
#Flag values that are > clip_lvl-sigma discrepant from the median.
good_inds = np.where((abs(through_stack - through_stack_median) /
through_stack_stddev <= clip_lvl))[0]
#Calculate the sigma-clipped mean and sigma-clipped stddev using good_inds.
s_c_mean = np.nanmean(through_stack[good_inds])
s_c_stddev = np.nanstd(through_stack[good_inds])
#Store the sigma-clipped mean as the master dark value for this pixel.
master_flat_lights_on[y, x] = s_c_mean
master_flat_lights_on_stddev[y, x] = s_c_stddev
#Make cube of the lights-off images
num_images = len(lights_off_files)
print('Reading in ', num_images, ' lights-off flat images.')
flat_lights_off_cube_raw = np.zeros([len(lights_off_files), 1024, 1024])
lights_off_std_devs = np.zeros(num_images)
print('')
print('Flat frame information')
print('-------------------------------------------------')
print('ID Mean Stddev Max Min')
print('-------------------------------------------------')
for j in range(len(lights_off_files)):
image_data = fits.open(dome_flat_raw_path /
lights_off_files[j])[0].data[0:1024, :]
header = fits.open(dome_flat_raw_path / lights_off_files[j])[0].header
if header['FILTNME2'] != band:
print(
'ERROR: {} taken in filter other than {}. Double check your date, try specifying start/stop file numbers, etc.'
.format(lights_off_files[j], band))
return
flat_lights_off_cube_raw[
j, :, :] = image_data #This line trims off the top two rows of the image, which are overscan.
lights_off_std_devs[j] = np.std(
image_data
) #Save standard deviation of flat images to identify flats with "ski jump" horizontal bars issue.
print(
str(j + 1) + ' ' + str(np.mean(image_data)) + ' ' +
str(np.std(image_data)) + ' ' + str(np.std(image_data)) +
' ' + str(np.amin(image_data)))
#Identify bad lights-on flat images (usually have bright horizontal bands at the top/bottom of images.)
vals, lo, hi = sigmaclip(lights_off_std_devs)
bad_locs = np.where((lights_off_std_devs < lo)
| (lights_off_std_devs > hi))[0]
good_locs = np.where((lights_off_std_devs > lo)
& (lights_off_std_devs < hi))[0]
if len(bad_locs > 0):
print(
'Found {} bad lights-off flats: {}.\nExcluding from combination step.\n'
.format(len(bad_locs),
np.array(lights_off_files)[bad_locs]))
time.sleep(2)
flat_lights_off_cube_raw = flat_lights_off_cube_raw[good_locs]
#For each pixel, calculate the mean, median, and standard deviation "through the stack" of lights off flat images.
lights_off_cube_shape = np.shape(flat_lights_off_cube_raw)
master_flat_lights_off = np.zeros(
(lights_off_cube_shape[1], lights_off_cube_shape[2]), dtype='float32')
master_flat_lights_off_stddev = np.zeros(
(lights_off_cube_shape[1], lights_off_cube_shape[2]), dtype='float32')
print('')
print('Combining the lights-off flats.')
print('......')
pbar = ProgressBar()
for x in pbar(range(lights_off_cube_shape[1])):
for y in range(lights_off_cube_shape[2]):
through_stack = flat_lights_off_cube_raw[:, y, x]
through_stack_median = np.nanmedian(through_stack)
through_stack_stddev = np.nanstd(through_stack)
#Flag values that are > clip_lvl-sigma discrepant from the median.
good_inds = np.where((abs(through_stack - through_stack_median) /
through_stack_stddev <= clip_lvl))[0]
#Calculate the sigma-clipped mean and sigma-clipped stddev using good_inds.
s_c_mean = np.nanmean(through_stack[good_inds])
s_c_stddev = np.nanstd(through_stack[good_inds])
#Store the sigma-clipped mean as the master dark value for this pixel.
master_flat_lights_off[y, x] = s_c_mean
master_flat_lights_off_stddev[y, x] = s_c_stddev
#Create the master flat
master_flat = master_flat_lights_on - master_flat_lights_off
master_flat_norm = np.nanmedian(master_flat)
master_flat = master_flat / master_flat_norm
#Create the master error flat
master_flat_error = np.sqrt(master_flat_lights_on_stddev**2 +
master_flat_lights_off_stddev**2)
master_flat_error = master_flat_error / master_flat_norm
#Ourput flat files.
output_filename = pines_path / ('Calibrations/Flats/Domeflats/' + band +
'/Master Flats/master_flat_' + band + '_' +
date + '.fits')
#Add some header keywords detailing the master_dark creation process.
hdu = fits.PrimaryHDU(master_flat)
hdu.header['HIERARCH DATE CREATED'] = datetime.utcnow().strftime(
'%Y-%m-%d') + 'T' + datetime.utcnow().strftime('%H:%M:%S')
#Now save to a file on your local machine.
print('')
print('Writing the file to {}'.format(output_filename))
#Check to see if other files of this name exist
# if os.path.exists(output_filename):
# print('')
# print('WARNING: This will overwrite {}!'.format(output_filename))
# flat_check = input('Do you want to continue? y/n: ')
# if flat_check == 'y':
# hdu.writeto(output_filename,overwrite=True)
# print('Wrote to {}!'.format(output_filename))
# else:
# print('Not overwriting!')
# else:
hdu.writeto(output_filename, overwrite=True)
print('Wrote to {}!'.format(output_filename))
print('')
if upload:
print('Beginning upload process to pines.bu.edu...')
print('Note, only PINES admins will be able to upload.')
print('')
sftp.chdir('/')
sftp.chdir('data/calibrations/Flats/Domeflats/' + band)
upload_name = 'master_flat_' + band + '_' + date + '.fits'
# if upload_name in sftp.listdir():
# print('WARNING: This will overwrite {} in pines.bu.edu:data/calibrations/Flats/Domeflats/{}/'.format(upload_name,band))
# upload_check = input('Do you want to continue? y/n: ')
# if upload_check == 'y':
# sftp.put(output_filename,upload_name)
# print('Uploaded to pines.bu.edu:data/calibrations/Flats/Domeflats/{}/ !'.format(band))
# else:
# print('Skipping upload!')
# else:
sftp.put(output_filename, upload_name)
print(
'Uploaded {} to pines.bu.edu:data/calibrations/Flats/Domeflats/{}/!'
.format(upload_name, band))
output_filename = pines_path / ('Calibrations/Flats/Domeflats/' + band +
'/Master Flats Stddev/master_flat_stddev_'
+ band + '_' + date + '.fits')
#Add some header keywords detailing the master_dark creation process.
hdu = fits.PrimaryHDU(master_flat_error)
hdu.header['HIERARCH DATE CREATED'] = datetime.utcnow().strftime(
'%Y-%m-%d') + 'T' + datetime.utcnow().strftime('%H:%M:%S')
#Now save to a file on your local machine.
#Check to see if other files of this name exist
# if os.path.exists(output_filename):
# print('')
# print('WARNING: This will overwrite {}!'.format(output_filename))
# flat_check = input('Do you want to continue? y/n: ')
# if flat_check == 'y':
# hdu.writeto(output_filename,overwrite=True)
# print('Wrote to {}!'.format(output_filename))
# else:
# print('Not overwriting!')
# else:
hdu.writeto(output_filename, overwrite=True)
print('Wrote to {}!'.format(output_filename))
if upload:
print('Uploading to pines.bu.edu...')
sftp.chdir('/')
sftp.chdir('data/calibrations/Flats Stddev/Domeflats/' + band)
upload_name = 'master_flat_stddev_' + band + '_' + date + '.fits'
# if upload_name in sftp.listdir():
# print('WARNING: This will overwrite {} in pines.bu.edu:data/calibrations/Flats/Domeflats/{}/'.format(upload_name,band))
# upload_check = input('Do you want to continue? y/n: ')
# if upload_check == 'y':
# sftp.put(output_filename,upload_name)
# print('Uploaded to pines.bu.edu:data/calibrations/Flats Stddev/Domeflats/{}/ !'.format(band))
# else:
# print('Skipping upload!')
# else:
sftp.put(output_filename, upload_name)
print(
'Uploaded {} to pines.bu.edu:data/calibrations/Flats Stddev/Domeflats/{}/!'
.format(upload_name, band))
print('')
if delete_raw:
files_to_delete = glob.glob(os.path.join(dome_flat_raw_path /
'*.fits'))
for j in range(len(files_to_delete)):
os.remove(files_to_delete[j])
print('dome_flat_field runtime: ', np.round((time.time() - t1) / 60, 1),
' minutes.')
print('Done!')
def simple_lightcurve(target,
sources,
centroided_sources,
phot_type='aper',
ref_set_choice=[],
plot_mode='combined'):
print('\nRunning simple_lightcurve().\n')
'''Authors:
Patrick Tamburo, Boston University, June 2020
Purpose:
Makes a "simple" lightcurve with each reference star weighted equally when creating the artificial comparison lightcurve.
Inputs:
target (str): the target's long name (e.g. '2MASS J12345678+1234567').
sources (pandas DataFrame): DataFrame with source names, and x/y positions in the source_detect_image. Output from ref_star_chooser.
centroided_sources (pandas DataFrame): Dataframe with source positions in every image. Output from centroider.
phot_type (str): 'aper' or 'psf'. Whether to use aperture or PSF phomometry NOTE: PSF photometry currently not implemented.
ref_set_choice (list): list of reference IDs to use to make the lightcurve, in case you want to exclude any.
plot_mode (str): 'combined' or 'separate'. 'combined' plots all nights in one figure, while 'separate' plots nights in separate figures.
Outputs:
Saves lightcurve plots to target's analysis directory.
TODO:
PSF photometry
Regression?
'''
def regression(flux, seeing, airmass, corr_significance=1e-5):
#Looks at correlations between seeing and airmass with the target flux.
#Takes those variables which are significantly correlated, and uses them in a linear regression to de-correlated the target flux.
#Use the seeing in the regression if it's significantly correlated
if pearsonr(seeing, flux)[1] < corr_significance:
use_seeing = True
else:
use_seeing = False
#Same thing for airmass
if pearsonr(airmass, flux)[1] < corr_significance:
use_airmass = True
else:
use_airmass = False
#Now set up the linear regression.
regr = linear_model.LinearRegression()
regress_dict = {}
#Add seeing, background, and airmass, if applicable.
if use_seeing:
key = 'seeing'
regress_dict[key] = seeing
if use_airmass:
key = 'airmass'
regress_dict[key] = airmass
#Finally, add target flux
regress_dict['flux'] = flux
#Get list of keys
keylist = list()
for i in regress_dict.keys():
keylist.append(i)
#Create data frame of regressors.
df = DataFrame(regress_dict, columns=keylist)
x = df[keylist[0:len(keylist) - 1]]
y = df['flux']
if np.shape(x)[1] > 0:
regr.fit(x, y)
#Now, define the model.
linear_regression_model = regr.intercept_
i = 0
#Add in the other regressors, as necessary. Can't think of a way of doing this generally, just use a bunch of ifs.
if (use_seeing) and (use_airmass):
linear_regression_model = linear_regression_model + regr.coef_[
0] * seeing + regr.coef_[1] * airmass
if (use_seeing) and not (use_airmass):
linear_regression_model = linear_regression_model + regr.coef_[
0] * seeing
if not (use_seeing) and (use_airmass):
linear_regression_model = linear_regression_model + regr.coef_[
0] * airmass
#Divide out the fit.
corrected_flux = flux / linear_regression_model
else:
#print('No regressors used.')
corrected_flux = flux
return corrected_flux
#plt.ion()
pines_path = pines_dir_check()
short_name = short_name_creator(target)
outlier_tolerance = 0.2 #If a reference > outlier_tolerance of its values above sigma clipping threshold, mark it as bad.
centroided_sources.columns = centroided_sources.keys().str.strip()
#Get list of photometry files for this target.
photometry_path = pines_path / ('Objects/' + short_name + '/' + phot_type +
'_phot/')
analysis_path = pines_path / ('Objects/' + short_name + '/analysis')
photometry_files = natsort.natsorted(
[x for x in photometry_path.glob('*.csv')])
num_refs = len(sources) - 1
#Loop over all photometry files in the aper_phot directory.
for i in range(len(photometry_files)):
#Load in the photometry data.
if phot_type == 'aper':
aperture_radius = float(str(photometry_files[i]).split('_')[-3])
phot_data = pines_log_reader(photometry_files[i])
#Remove entries that have NaN's for flux values.
for j in range(len(sources['Name'])):
name = sources['Name'][j]
phot_data[name + ' Flux'] = phot_data[name + ' Flux'].astype(float)
phot_data[name +
' Flux Error'] = phot_data[name +
' Flux Error'].astype(float)
#Get target interpolation warnings.
targ_interp_flags = np.array(phot_data[short_name +
' Interpolation Flag'])
#Get times of exposures.
times = np.array(phot_data['Time JD'])
seeing = np.array(phot_data['Seeing'])
airmass = np.array(phot_data['Airmass'])
background = np.array(phot_data[sources['Name'][0] + ' Background'])
#Convert to datetimes for plotting purposes.
dts = np.array(
[julian.from_jd(times[i], fmt='jd') for i in range(len(times))])
#Get the target's flux and background
targ_flux = np.array(phot_data[short_name + ' Flux'])
targ_flux_err = np.array(phot_data[short_name + ' Flux Error'])
#Get the reference stars' fluxes and backgrounds.
ref_flux = np.zeros((num_refs, len(phot_data)))
ref_flux_err = np.zeros((num_refs, len(phot_data)))
for j in range(0, num_refs):
ref_flux[j, :] = phot_data['Reference ' + str(j + 1) + ' Flux']
ref_flux_err[j, :] = phot_data['Reference ' + str(j + 1) +
' Flux Error']
#Discard variable stars.
#values, clow, chigh = sigmaclip(ref_flux[j], low=2.5, high=2.5)
# if (len(phot_data) - len(values)) > (int(outlier_tolerance * len(phot_data))):
# print('Have to add flagging bad refs.')
closest_ref = np.where(
abs(np.nanmean(ref_flux, axis=1) - np.nanmean(targ_flux)) == min(
abs(np.nanmean(ref_flux, axis=1) -
np.nanmean(targ_flux))))[0][0]
#Split data up into individual nights.
night_inds = night_splitter(times)
num_nights = len(night_inds)
#if plot_mode == 'combined':
# fig, axis = plt.subplots(nrows=1, ncols=num_nights, figsize=(16, 5))
#colors = ['tab:blue', 'tab:orange', 'tab:green', 'tab:red', 'tab:purple', 'tab:brown', 'tab:pink', 'tab:gray', 'tab:olive', 'tab:cyan']
#Get the time range of each night. Set each plot panel's xrange according to the night with the longest time. Makes seeing potential variability signals easier.
night_lengths = np.zeros(num_nights)
for j in range(num_nights):
inds = night_inds[j]
night_lengths[j] = times[inds][-1] - times[inds][0]
longest_night = max(night_lengths)
longest_night_hours = np.ceil(longest_night * 24)
global line_list, filename_list
line_list = []
filename_list = []
for j in range(num_nights):
#if plot_mode == 'separate':
# fig, ax = plt.subplots(1, 1, figsize=(16,5))
#else:
# ax = axis[j]
#if phot_type =='aper':
# fig.suptitle(short_name, fontsize=16)
j += 1
#if j == 1:
# ax.set_ylabel('Normalized Flux', fontsize=16)
#ax.set_xlabel('Time (UT)', fontsize=16)
inds = night_inds[j - 1]
filename_list.append(
np.array([phot_data['Filename'][z] for z in inds]))
alc = np.zeros(len(inds))
#Normalize reference lightcurves
#TODO: Each night should be normalized separately.
for k in range(num_refs):
ref_flux[k][inds] = ref_flux[k][inds] / np.nanmedian(
ref_flux[k][inds])
for k in range(len(inds)):
#Do a sigma clip on normalized references to avoid biasing median.
###values, clow, chigh = sigmaclip(ref_flux[:,inds[k]][~np.isnan(ref_flux[:,inds[k]])], low=1.5, high=1.5)
###alc[k] = np.median(values)
avg, med, std = sigma_clipped_stats(
ref_flux[:, inds[k]][~np.isnan(ref_flux[:, inds[k]])],
sigma=1.5)
alc[k] = med
#Correct the target lightcurve using the alc.
alc = alc / np.nanmedian(alc)
targ_flux_norm = targ_flux[inds] / np.nanmedian(targ_flux[inds])
targ_corr = targ_flux_norm / alc
targ_corr = targ_corr / np.nanmedian(targ_corr)
#Correct the example reference lightcurve using the alc.
ref_corr_norm = ref_flux[closest_ref][inds] / np.nanmedian(
ref_flux[closest_ref][inds])
ref_corr = ref_corr_norm / alc
ref_corr = ref_corr / np.nanmedian(ref_corr)
#Plot the target and reference lightcurves.
#t_plot, = ax.plot(dts[inds], targ_corr, '.', color=colors[i])
#line_list.append(t_plot)
#myFmt = mdates.DateFormatter('%H:%M')
#ax.xaxis.set_major_formatter(myFmt)
#fig.autofmt_xdate()
#Do sigma clipping on the corrected lightcurve to get rid of outliers (from clouds, bad target centroid, cosmic rays, etc.)
###vals, lo, hi = sigmaclip(targ_corr, low=2.5, high=2.5)
avg, med, std = sigma_clipped_stats(targ_corr, sigma=3)
bad_vals = np.where((targ_corr > med + 5 * std)
| (targ_corr < med - 5 * std))[0]
good_vals = np.where((targ_corr < med + 5 * std)
& (targ_corr > med - 5 * std))[0]
vals = targ_corr[good_vals]
#if len(bad_vals) != 0:
# plt.plot(dts[inds][bad_vals], targ_corr[bad_vals], marker='x',color='r', mew=1.8, ms=7, zorder=0, ls='')
blocks = block_splitter(times[inds], bad_vals)
bin_times = np.zeros(len(blocks))
bin_fluxes = np.zeros(len(blocks))
bin_errs = np.zeros(len(blocks))
bin_dts = []
for k in range(len(blocks)):
try:
bin_times[k] = np.mean(times[inds][blocks[k]])
#vals, hi, lo = sigmaclip(targ_corr[blocks[k]],high=3,low=3) #Exclude outliers.
bin_fluxes[k] = np.mean(targ_corr[blocks[k]])
bin_errs[k] = np.std(targ_corr[blocks[k]]) / np.sqrt(
len(targ_corr[blocks[k]]))
bin_dts.append(julian.from_jd(bin_times[k], fmt='jd'))
except:
pdb.set_trace()
bin_dts = np.array(bin_dts)
#ax.errorbar(bin_dts, bin_fluxes, yerr=bin_errs, marker='o', color='k',zorder=3, ls='')
#Draw the y=1 and 5-sigma detection threshold lines.
#ax.axhline(y=1, color='r', lw=2, zorder=0)
#ax.axhline(1-5*np.median(bin_errs), zorder=0, lw=2, color='k', ls='--', alpha=0.4)
#Set the y-range so you can see the 5-sigma detection line.
#ax.set_ylim(0.9, 1.1)
#Set the x-range to be the same for all nights.
#ax.set_xlim(julian.from_jd(times[inds][0]-0.025, fmt='jd'), julian.from_jd(times[inds][0]+longest_night+0.025, fmt='jd'))
#ax.grid(alpha=0.2)
#ax.set_title(phot_data['Time UT'][inds[0]].split('T')[0], fontsize=14)
#ax.tick_params(labelsize=12)
#print('average seeing, night {}: {}'.format(j, np.mean(seeing[inds])))
#pdb.set_trace()
#print('pearson correlation between target and closest ref: {}'.format(pearsonr(targ_corr[good_vals], ref_corr[good_vals])))
#print(np.mean(bin_errs))
#print('')
#fig.tight_layout(rect=[0, 0.03, 1, 0.93])
#Output the simple lc data to a csv.
time_save = times
flux_save = targ_corr
flux_err_save = np.zeros(len(flux_save)) + np.std(targ_corr)
output_dict = {
'Time': time_save,
'Flux': flux_save,
'Flux Error': flux_err_save
}
output_df = pd.DataFrame(data=output_dict)
if phot_type == 'aper':
output_filename = analysis_path / (
short_name + '_simple_lc_aper_phot_' +
str(np.round(aperture_radius, 1)) + '_pix.csv')
print('\nSaving to {}.\n'.format(output_filename))
output_df.to_csv(output_filename)
elif phot_type == 'psf':
print("ERROR: Need to create flux output for PSF photometry.")
return
def variable_aper_phot(target,
centroided_sources,
multiplicative_factors,
an_in=12.,
an_out=30.,
plots=False,
gain=8.21,
qe=0.9,
plate_scale=0.579):
pines_path = pines_dir_check()
short_name = short_name_creator(target)
#Remove any leading/trailing spaces in the column names.
centroided_sources.columns = centroided_sources.columns.str.lstrip()
centroided_sources.columns = centroided_sources.columns.str.rstrip()
#Get list of reduced files for target.
reduced_path = pines_path / ('Objects/' + short_name + '/reduced')
reduced_filenames = natsort.natsorted(
[x.name for x in reduced_path.glob('*red.fits')])
reduced_files = np.array([reduced_path / i for i in reduced_filenames])
#Get source names.
source_names = get_source_names(centroided_sources)
#Get seeing.
seeing = np.array(centroided_sources['Seeing'])
#Loop over multiplicative factors
for i in range(len(multiplicative_factors)):
fact = multiplicative_factors[i]
print(
'Doing variable aperture photometry for {}, multiplicative seeing factor = {}, inner annulus radius = {} pix, outer annulus radius = {} pix.'
.format(target, fact, an_in, an_out))
#Declare a new dataframe to hold the information for all targets for this aperture.
columns = [
'Filename', 'Time UT', 'Time JD UTC', 'Time BJD TDB', 'Airmass',
'Seeing'
]
for j in range(0, len(source_names)):
columns.append(source_names[j] + ' Flux')
columns.append(source_names[j] + ' Flux Error')
columns.append(source_names[j] + ' Background')
columns.append(source_names[j] + ' Interpolation Flag')
var_df = pd.DataFrame(index=range(len(reduced_files)), columns=columns)
output_filename = pines_path / (
'Objects/' + short_name + '/aper_phot/' + short_name +
'_variable_aper_phot_' + str(float(fact)) + '_seeing_factor.csv')
#Loop over all images.
pbar = ProgressBar()
for j in pbar(range(len(reduced_files))):
data = fits.open(reduced_files[j])[0].data
#Read in some supporting information.
log_path = pines_path / (
'Logs/' + reduced_files[j].name.split('.')[0] + '_log.txt')
log = pines_log_reader(log_path)
log_ind = np.where(
log['Filename'] == reduced_files[j].name.split('_')[0] +
'.fits')[0][0]
header = fits.open(reduced_files[j])[0].header
date_obs = header['DATE-OBS']
#Catch a case that can cause datetime strptime to crash; Mimir headers sometimes have DATE-OBS with seconds specified as 010.xx seconds, when it should be 10.xx seconds.
if len(date_obs.split(':')[-1].split('.')[0]) == 3:
date_obs = date_obs.split(':')[0] + ':' + date_obs.split(
':')[1] + ':' + date_obs.split(':')[-1][1:]
if date_obs.split(':')[-1] == '60.00':
date_obs = date_obs.split(':')[0] + ':' + str(
int(date_obs.split(':')[1]) + 1) + ':00.00'
#Keep a try/except clause here in case other unknown DATE-OBS formats pop up.
try:
date = datetime.datetime.strptime(date_obs,
'%Y-%m-%dT%H:%M:%S.%f')
except:
print(
'Header DATE-OBS format does not match the format code in strptime! Inspect/correct the DATE-OBS value.'
)
pdb.set_trace()
#Get the closest date master_dark_stddev image for this exposure time.
#We'll use this to measure read noise and dark current.
date_str = date_obs.split('T')[0].replace('-', '')
master_dark_stddev = master_dark_stddev_chooser(
pines_path / ('Calibrations/Darks/Master Darks Stddev/'),
header)
days = date.day + hmsm_to_days(date.hour, date.minute, date.second,
date.microsecond)
jd = date_to_jd(date.year, date.month, days)
var_df['Filename'][j] = reduced_files[j].name
var_df['Time UT'][j] = header['DATE-OBS']
var_df['Time JD UTC'][j] = jd
var_df['Time BJD TDB'][j] = jd_utc_to_bjd_tdb(
jd, header['TELRA'], header['TELDEC'])
var_df['Airmass'][j] = header['AIRMASS']
var_df['Seeing'][j] = log['X seeing'][np.where(
log['Filename'] == reduced_files[j].name.split('_')[0] +
'.fits')[0][0]]
#If the shift quality has been flagged, skip this image.
if log['Shift quality flag'].iloc[log_ind] == 1:
continue
#Get the source positions in this image.
positions = []
for k in range(len(source_names)):
positions.append(
(centroided_sources[source_names[k] + ' Image X'][j],
centroided_sources[source_names[k] + ' Image Y'][j]))
#Create an aperture centered on this position with radius (in pixels) of (seeing*multiplicative_factor[j])/plate_scale.
try:
apertures = CircularAperture(positions,
r=(seeing[j] * fact) /
plate_scale)
except:
pdb.set_trace()
#Create an annulus centered on this position.
annuli = CircularAnnulus(positions, r_in=an_in, r_out=an_out)
photometry_tbl = iraf_style_photometry(apertures, annuli,
data * gain,
master_dark_stddev * gain,
header, var_df['Seeing'][j])
for k in range(len(photometry_tbl)):
var_df[source_names[k] +
' Flux'][j] = photometry_tbl['flux'][k]
var_df[source_names[k] +
' Flux Error'][j] = photometry_tbl['flux_error'][k]
var_df[source_names[k] +
' Background'][j] = photometry_tbl['background'][k]
var_df[source_names[k] + ' Interpolation Flag'][j] = int(
photometry_tbl['interpolation_flag'][k])
#Write output to file.
print(
'Saving multiplicative factor = {} variable aperture photometry output to {}.'
.format(fact, output_filename))
print('')
with open(output_filename, 'w') as f:
for j in range(len(var_df)):
#Write in the header.
if j == 0:
f.write(
'{:>21s}, {:>22s}, {:>17s}, {:>17s}, {:>7s}, {:>7s}, '.
format('Filename', 'Time UT', 'Time JD UTC',
'Time BJD TDB', 'Airmass', 'Seeing'))
for k in range(len(source_names)):
if k != len(source_names) - 1:
f.write(
'{:>22s}, {:>28s}, {:>28s}, {:>34s}, '.format(
source_names[k] + ' Flux',
source_names[k] + ' Flux Error',
source_names[k] + ' Background',
source_names[k] + ' Interpolation Flag'))
else:
f.write(
'{:>22s}, {:>28s}, {:>28s}, {:>34s}\n'.format(
source_names[k] + ' Flux',
source_names[k] + ' Flux Error',
source_names[k] + ' Background',
source_names[k] + ' Interpolation Flag'))
#Write in Filename, Time UT, Time JD, Airmass, Seeing values.
format_string = '{:21s}, {:22s}, {:17.9f}, {:17.9f}, {:7.2f}, {:7.1f}, '
#If the seeing value for this image is 'nan' (a string), convert it to a float.
#TODO: Not sure why it's being read in as a string, fix that.
if type(var_df['Seeing'][j]) == str:
var_df['Seeing'][j] = float(var_df['Seeing'][j])
#Do a try/except clause for writeout, in case it breaks in the future.
try:
f.write(
format_string.format(var_df['Filename'][j],
var_df['Time UT'][j],
var_df['Time JD UTC'][j],
var_df['Time BJD TDB'][j],
var_df['Airmass'][j],
var_df['Seeing'][j]))
except:
print(
'Writeout failed! Inspect quantities you are trying to write out.'
)
pdb.set_trace()
#Write in Flux, Flux Error, and Background values for every source.
for i in range(len(source_names)):
if i != len(source_names) - 1:
format_string = '{:22.5f}, {:28.5f}, {:28.5f}, {:34d}, '
else:
format_string = '{:22.5f}, {:28.5f}, {:28.5f}, {:34d}\n'
try:
f.write(
format_string.format(
var_df[source_names[i] + ' Flux'][j],
var_df[source_names[i] + ' Flux Error'][j],
var_df[source_names[i] + ' Background'][j],
var_df[source_names[i] +
' Interpolation Flag'][j]))
except:
if i != len(source_names) - 1:
format_string = '{:22.5f}, {:28.5f}, {:28.5f}, {:34f}, '
else:
format_string = '{:22.5f}, {:28.5f}, {:28.5f}, {:34f}\n'
f.write(
format_string.format(
var_df[source_names[i] + ' Flux'][j],
var_df[source_names[i] + ' Flux Error'][j],
var_df[source_names[i] + ' Background'][j],
var_df[source_names[i] +
' Interpolation Flag'][j]))
print('')
return
def drift_rate(target):
plt.ion()
pines_path = pines_dir_check()
short_name = short_name_creator(target)
source_centroids = pd.read_csv(
pines_path / ('Objects/' + short_name +
'/sources/target_and_references_centroids.csv'))
source_centroids.columns = source_centroids.columns.str.strip()
x = source_centroids[short_name + ' X']
y = source_centroids[short_name + ' Y']
z = np.sqrt(x**2 + y**2)
reduced_images = natsort.natsorted(
list((pines_path /
('Objects/' + short_name + '/reduced/')).rglob('*.fits')))
if len(reduced_images) != len(x):
raise RuntimeError(
'Number of reduced images does not match number of centroids.')
dates = []
for i in range(len(reduced_images)):
file = reduced_images[i]
date_obs = fits.open(file)[0].header['DATE-OBS']
dates.append(
datetime.datetime.strptime(date_obs, '%Y-%m-%dT%H:%M:%S.%f'))
dates = np.array(dates)
unique_days = natsort.natsorted(
list({(i.day, i.month, i.year)
for i in dates}))
days = [(dates[i].day, dates[i].month, dates[i].year)
for i in range(len(dates))]
colors = ['tab:blue', 'tab:orange', 'tab:green']
fig, ax = plt.subplots(nrows=3,
ncols=len(unique_days),
figsize=(17, 8),
sharey=True)
for i in range(len(unique_days)):
day = unique_days[i]
truth_arr = [days[i] == day for i in range(len(days))]
date_locs = np.where(truth_arr)[0]
ax[i, 0].plot(dates[date_locs],
np.gradient(x[date_locs]),
marker='o',
ls='',
color=colors[0],
alpha=0.3)
smoothed = scipy.ndimage.filters.uniform_filter(np.gradient(
x[date_locs]),
size=5)
ax[i, 0].plot(dates[date_locs],
smoothed,
lw=2,
color=colors[0],
zorder=0)
for label in ax[i, 0].get_xticklabels():
label.set_rotation(30)
label.set_horizontalalignment('right')
ax[i, 1].plot(dates[date_locs],
np.gradient(y[date_locs]),
marker='o',
ls='',
color=colors[1],
alpha=0.3)
smoothed = scipy.ndimage.filters.uniform_filter(np.gradient(
y[date_locs]),
size=5)
ax[i, 1].plot(dates[date_locs],
smoothed,
lw=2,
color=colors[1],
zorder=0)
for label in ax[i, 1].get_xticklabels():
label.set_rotation(30)
label.set_horizontalalignment('right')
ax[i, 2].plot(dates[date_locs],
np.gradient(z[date_locs]),
marker='o',
ls='',
color=colors[2],
alpha=0.3)
smoothed = scipy.ndimage.filters.uniform_filter(np.gradient(
z[date_locs]),
size=5)
ax[i, 2].plot(dates[date_locs],
smoothed,
lw=2,
color=colors[2],
zorder=0)
for label in ax[i, 2].get_xticklabels():
label.set_rotation(30)
label.set_horizontalalignment('right')
ax[i, 0].grid(True, alpha=0.4)
ax[i, 1].grid(True, alpha=0.4)
ax[i, 2].grid(True, alpha=0.4)
if i == 0:
ax[i, 0].set_title('dx/dt', color=colors[0], fontsize=16)
ax[i, 1].set_title('dy/dt', color=colors[1], fontsize=16)
ax[i, 2].set_title('dz/dt', color=colors[2], fontsize=16)
plt.tight_layout()
pdb.set_trace()
def log_updater(date, sftp, shift_tolerance=30, upload=False):
'''
Authors:
Patrick Tamburo, Boston University, January 2021
Purpose:
Updates x_shift and y_shift measurements from a PINES log. These shifts are measured using *full* resolution images, while at the telescope,
we use *half* resolution images (to save time between exposures). By measuring on full-res images, we get more accurate shifts, which allows
us to determine centroids more easily.
Inputs:
date (str): the UT date of the log whose shifts you want to update in YYYYMMDD format, e.g. '20151110'
sftp (pysftp connection): sftp connection to the PINES server
shift_tolerance (float): the maximum distance an x/y shift can be before shifts will be flagged as poor quality.
upload (bool): whether or not to push the updated log to the PINES server (only admins can do this)
Outputs:
Writes updated log file to disk.
TODO:
Re-measure seeing?
FIXME:
'''
def tie_sigma(model):
return model.x_stddev_1
def guide_star_seeing(subframe):
# subframe = subframe - np.median(subframe)
subframe = subframe - np.percentile(subframe,5)
sub_frame_l = int(np.shape(subframe)[0])
y, x = np.mgrid[:sub_frame_l, :sub_frame_l]
# Fit with constant, bounds, tied x and y sigmas and outlier rejection:
gaussian_init = models.Const2D(0.0) + models.Gaussian2D(subframe[int(sub_frame_l/2),int(sub_frame_l/2)],int(sub_frame_l/2),int(sub_frame_l/2),8/2.355,8/2.355,0)
gaussian_init.x_stddev_1.min = 1.0/2.355
gaussian_init.x_stddev_1.max = 20.0/2.355
gaussian_init.y_stddev_1.min = 1.0/2.355
gaussian_init.y_stddev_1.max = 20.0/2.355
gaussian_init.y_stddev_1.tied = tie_sigma
gaussian_init.theta_1.fixed = True
fit_gauss = fitting.FittingWithOutlierRemoval(fitting.LevMarLSQFitter(),sigma_clip,niter=3,sigma=3.0)
# gaussian, mask = fit_gauss(gaussian_init, x, y, subframe)
gain = 8.21 #e per ADU
read_noise = 2.43 #ADU
weights = gain / np.sqrt(np.absolute(subframe)*gain + (read_noise*gain)**2) #1/sigma for each pixel
gaussian, mask = fit_gauss(gaussian_init, x, y, subframe, weights)
fwhm_x = 2.355*gaussian.x_stddev_1.value
fwhm_y = 2.355*gaussian.y_stddev_1.value
x_seeing = fwhm_x * 0.579
y_seeing = fwhm_y * 0.579
return(x_seeing,y_seeing)
pines_path = pines_dir_check()
log_path = pines_path/('Logs/'+date+'_log.txt')
#Begin by checking filenames, making sure they're in sequential order, and that there is only one entry for each.
log_out_of_order_fixer(log_path, sftp)
log = pines_log_reader(log_path) #Get telescope log shifts.
myfile = open(log_path, 'r')
lines = myfile.readlines()
myfile.close()
#Now loop over all files in the log, measure shifts in each file and update the line in the log.
for i in range(len(log)):
if (log['Target'][i].lower() != 'flat') & (log['Target'][i].lower() != 'skyflat') & (log['Target'][i].lower() != 'supersky') & (log['Target'][i].lower() != 'dark') & (log['Target'][i].lower() != 'bias') & (log['Target'][i].lower() != 'dummy') & (log['Post-processing flag'][i] != 1):
filename = log['Filename'][i].split('.fits')[0]+'_red.fits'
target = log['Target'][i]
short_name = short_name_creator(target)
image_path = pines_path/('Objects/'+short_name+'/reduced/'+filename)
#Figure out which file you're looking at and its position in the log.
log_ind = np.where(log['Filename'] == filename.split('_')[0]+'.fits')[0][0]
#Measure the shifts and get positions of targets.
(measured_x_shift, measured_y_shift, source_x, source_y, check_image) = shift_measurer(target, filename, sftp)
if (abs(measured_x_shift) > shift_tolerance) or (abs(measured_y_shift) > shift_tolerance):
print('Shift greater than {} pixels measured for {} in {}.'.format(shift_tolerance, short_name, image_path.name))
print('Inspect manually.')
shift_quality_flag = 1
elif np.isnan(measured_x_shift) or np.isnan(measured_y_shift):
raise RuntimeError('Found nans for shifts!')
shift_quality_flag = 1
else:
shift_quality_flag = 0
#Measure the seeing.
guide_star_cut = np.where((source_x > 50) & (source_x < 975) & (source_y > 50) & (source_y < 975))[0]
if len(guide_star_cut) != 0:
x_seeing_array = []
y_seeing_array = []
for guide_star_ind in guide_star_cut:
guide_star_x_int = int(source_x[guide_star_ind])
guide_star_y_int = int(source_y[guide_star_ind])
guide_star_subframe = check_image[guide_star_y_int-15:guide_star_y_int+15,guide_star_x_int-15:guide_star_x_int+15]
(x_seeing,y_seeing) = guide_star_seeing(guide_star_subframe)
#Cut unrealistic values/saturated stars.
if x_seeing > 1.2 and x_seeing < 7.0:
x_seeing_array.append(x_seeing)
y_seeing_array.append(y_seeing)
x_seeing = np.nanmedian(x_seeing_array)
y_seeing = np.nanmedian(y_seeing_array)
else:
#Default to the average PINES value if no sources were found for guiding.
x_seeing = 2.6
y_seeing = 2.6
print('Log line {} of {}.'.format(i+1, len(log)))
print('Measured x shift: {:4.1f}, measured y shift: {:4.1f}'.format(measured_x_shift, measured_y_shift))
print('Measured seeing: {:4.1f}'.format(x_seeing))
print('')
#Overwrite the telescope's logged shifts and seeing values with the new measurements.
log['X shift'][log_ind] = str(np.round(measured_x_shift,1))
log['Y shift'][log_ind] = str(np.round(measured_y_shift,1))
log['X seeing'][log_ind] = str(np.round(x_seeing, 1))
log['Y seeing'][log_ind] = str(np.round(y_seeing, 1))
#Grab entries for log line.
filename = log['Filename'][log_ind]
log_date = log['Date'][log_ind]
target_name = log['Target'][log_ind]
filter_name = log['Filt.'][log_ind]
exptime = log['Exptime'][log_ind]
airmass = log['Airmass'][log_ind]
x_shift = log['X shift'][log_ind]
y_shift = log['Y shift'][log_ind]
x_seeing = log['X seeing'][log_ind]
y_seeing = log['Y seeing'][log_ind]
post_processing_flag = 1
#Generate line of log text following the PINES telescope log format.
log_text = pines_logging(filename, log_date, target_name, filter_name, exptime, airmass, x_shift, y_shift, x_seeing, y_seeing, post_processing_flag, shift_quality_flag)
#Overwrite the line with the new shifts.
line_ind = log_ind + 1
lines[line_ind] = log_text
#Update the log on disk.
with open(log_path, 'w') as f:
for line in lines:
f.write(line)
elif (log['Post-processing flag'][i] == 1):
print('File already post-processed, skipping. {} of {}'.format(i+1, len(log)))
else:
print('File not a science target, skipping. {} of {}.'.format(i+1, len(log)))
if upload:
sftp.chdir('/data/logs/')
print('Uploading to /data/logs/{}_log.txt.'.format(date))
sftp.put(log_path,date+'_log.txt')
return
def reduce(target_name, upload=False, delete_raw=False, delete_reduced=False, sftp='', manual_flat_path='', manual_dark_path='', manual_bpm_path='', linearity_correction=False, linearity_correction_degree=5):
t1 = time.time()
print('')
if (upload is True) and (sftp == ''):
print('ERROR: You must pass an sftp connection if you want to upload reduced files to pines.bu.edu.!')
return
pines_path = pines_dir_check()
short_name = short_name_creator(target_name)
#Paths
raw_path = pines_path/('Objects/'+short_name+'/raw')
raw_files = [Path(i) for i in natsort.natsorted(glob.glob(os.path.join(raw_path,'*.fits')))] #Natsort sorts things how you expect them to be sorted.
dark_path = pines_path/('Calibrations/Darks')
reduced_path = pines_path/('Objects/'+short_name+'/reduced')
flats_path = pines_path/('Calibrations/Flats/Domeflats')
bpm_path = pines_path/('Calibrations/Bad Pixel Masks')
#Now begin the loop to load and reduce the raw science data
print("Reducing data for {}.".format(target_name))
print('Note: any reduced data already in reduced directory will be not be re-reduced.')
pbar = ProgressBar()
for i in pbar(range(np.size(raw_files))):
target_filename = reduced_path/(raw_files[i].name.split('.fits')[0]+'_red.fits')
if target_filename.exists():
#print('{} already in reduced directory, skipping.'.format(target_filename.name))
continue
hdulist = fits.open(raw_files[i])
header = hdulist[0].header
try:
frame_raw = fits.open(raw_files[i])[0].data.astype('float32')
except:
pdb.set_trace()
frame_raw = frame_raw[0:1024,:] #Cuts off 2 rows of overscan (?) pixels
#Add a flag to the header to check if background is near saturation
if sigma_clipped_stats(frame_raw)[1] > 3800:
sat_flag = 1
else:
sat_flag = 0
#Load in the dark/flat files. If a manual path is not provided, choose the reduction image that is closest in time to when the image was taken.
if manual_flat_path == '':
master_flat, master_flat_name = master_flat_chooser(flats_path, header)
else:
master_flat = fits.open(manual_flat_path)[0].data
master_flat_name = manual_flat_path.name
if manual_dark_path == '':
master_dark, master_dark_name = master_dark_chooser(dark_path, header)
else:
master_dark = fits.open(manual_dark_path)[0].data
master_dark_name = manual_dark_path.name
if manual_bpm_path == '':
bad_pixel_mask, bad_pixel_mask_name = bpm_chooser(bpm_path, header)
else:
bad_pixel_mask = fits.open(manual_bpm_path)[0].data
bad_pixel_mask_name = manual_bpm_path.name
#If linearity_correction is set, correct the pixels for linearity.
if linearity_correction:
poly_coeffs_path = pines_path/('Calibrations/Linearity/meas2corr_poly_coeffs_deg_{}_no_redchi.fits'.format(linearity_correction_degree))
hdu_list = fits.open(poly_coeffs_path)
meas2corr_poly_coeffs = hdu_list[0].data[0:1024,:,:]
degree = meas2corr_poly_coeffs.shape[2]
frame_raw[np.where(bad_pixel_mask == 1)] = np.nan
measured_counts = frame_raw - master_dark
#Make an array of corrected counts, and correct using the polynomial coefficients form meas2corr_poly_coeffs.fits
corrected_counts = np.zeros((1024,1024), dtype='float32')
for i in np.arange(degree):
corrected_counts += meas2corr_poly_coeffs[:,:,i] * measured_counts**i
frame_red = corrected_counts
else:
#Reduce the image.
frame_red = (frame_raw - master_dark)/master_flat
frame_red = frame_red.astype('float32')
#Set bad pixels to NaNs.
frame_red[np.where(bad_pixel_mask == 1)] = np.nan
# pdb.set_trace()
# #Do a background model subtraction
# frame_red = bg_2d(frame_red)
#Store some parameters in the reduced file's header.
#Naming convention follows https://docs.astropy.org/en/stable/io/fits/usage/headers.html.
header['HIERARCH DATE REDUCED'] = datetime.utcnow().strftime('%Y-%m-%d')+'T'+datetime.utcnow().strftime('%H:%M:%S')
header['HIERARCH MASTER DARK'] = master_dark_name
header['HIERARCH MASTER FLAT'] = master_flat_name
header['HIERARCH BAD PIXEL MASK'] = bad_pixel_mask_name
header['HIERARCH SATURATION FLAG'] = sat_flag
if not os.path.exists(target_filename):
fits.writeto(target_filename, frame_red, header)
#print("Reducing {}: {} of {}, band = {}, exptime = {} s, dark = {}, flat = {}".format(raw_files[i].name, str(i+1), str(np.size(raw_files)), header['FILTNME2'], header['EXPTIME'], master_dark_name, master_flat_name))
if upload:
print('')
print('Beginning upload process to pines.bu.edu...')
print('NOTE: Only PINES admins are able to upload.')
print('WARNING: If these reduced images already exist on the PINES server, they will be overwritten!')
time.sleep(1)
sftp.chdir('/data/reduced/mimir/')
files_to_upload = np.array(natsort.natsorted(np.array([x for x in reduced_path.glob('*.fits')])))
print('Uploading reduced {} data to the PINES server!'.format(target_name))
pbar = ProgressBar()
for i in pbar(range(len(files_to_upload))):
file = files_to_upload[i]
night_name = files_to_upload[i].name.split('.')[0]
for dir_change in sftp.listdir():
sftp.chdir(dir_change)
nights = sftp.listdir()
if night_name in nights:
ind = np.where(np.array(nights) == night_name)[0][0]
break
sftp.chdir('..')
if nights[ind] != night_name:
print('ERROR: the date of the file you want to upload does not match the date directory where the program wants to upload it.')
pdb.set_trace()
#print('Uploading to {}/{}, {} of {}'.format(sftp.getcwd(),nights[ind]+'/'+file.name, i+1,len(files_to_upload)))
sftp.put(file,nights[ind]+'/'+file.name)
sftp.chdir('..')
if delete_raw:
files_to_delete = glob.glob(os.path.join(raw_path/'*.fits'))
for j in range(len(files_to_delete)):
os.remove(files_to_delete[j])
if delete_reduced:
files_to_delete = glob.glob(os.path.join(reduced_path/'*.fits'))
for j in range(len(files_to_delete)):
os.remove(files_to_delete[j])
def relative_cutout_position_plot(target, centroided_sources):
pines_path = pines_dir_check()
short_name = short_name_creator(target)
#Get plot style parameters.
title_size, axis_title_size, axis_ticks_font_size, legend_font_size = plot_style(
)
#Get list of souce names in the centroid output.
source_names = get_source_names(centroided_sources)
centroided_sources.columns = centroided_sources.keys().str.strip()
#Get times from the centroid output and split them by night.
times_full = np.array(centroided_sources['Time (JD UTC)'])
night_inds = night_splitter(times_full)
num_nights = len(night_inds)
times_nights = [times_full[night_inds[i]] for i in range(num_nights)]
standard_x = standard_x_range(times_nights)
#Get the box size (I don't like that this is being determined by using the mean of the data...output it from centroider?)
box_w = int(
np.round(
2 * np.nanmean(
np.array(centroided_sources['Reference 1 Cutout X'],
dtype='float')), 0))
fig, ax = plt.subplots(nrows=2,
ncols=num_nights,
figsize=(17, 9),
sharey=True)
plt.subplots_adjust(left=0.07,
hspace=0.05,
wspace=0.05,
top=0.92,
bottom=0.17)
markers = ['+', 'x', '*', 'X']
for j in range(num_nights):
inds = night_inds[j]
if j == 0:
ax[0, j].set_ylabel('Cutout X Position', fontsize=axis_title_size)
ax[1, j].set_ylabel('Cutout Y Position', fontsize=axis_title_size)
for i in range(len(source_names)):
cutout_x = np.array(centroided_sources[source_names[i] +
' Cutout X'][inds],
dtype='float')
cutout_y = np.array(centroided_sources[source_names[i] +
' Cutout Y'][inds],
dtype='float')
if i == 0:
marker = 'o'
label = 'Target'
else:
marker = markers[(i - 1) % len(markers)]
label = 'Ref. ' + str(i)
ax[0, j].plot(times_nights[j],
cutout_x,
marker=marker,
label=label,
linestyle='')
ax[1, j].plot(times_nights[j],
cutout_y,
marker=marker,
linestyle='')
ax[0, j].tick_params(labelsize=axis_ticks_font_size)
ax[0, j].set_xticklabels([])
ax[0, j].axhline(box_w / 2,
zorder=0,
color='r',
label='Center pix.',
lw=2)
ax[0, j].set_xlim(
np.mean(times_nights[j]) - standard_x / 2,
np.mean(times_nights[j]) + standard_x / 2)
ax[0, j].grid(alpha=0.2)
ax[1, j].tick_params(labelsize=axis_ticks_font_size)
ax[1, j].axhline(box_w / 2,
zorder=0,
color='r',
label='Center pix.',
lw=2)
ax[1, j].set_xlim(
np.mean(times_nights[j]) - standard_x / 2,
np.mean(times_nights[j]) + standard_x / 2)
ax[1, j].set_xlabel('Time (JD UTC)', fontsize=axis_title_size)
ax[1, j].grid(alpha=0.2)
if j == num_nights - 1:
ax[0, j].legend(bbox_to_anchor=(1.01, 1.0),
fontsize=legend_font_size)
plt.suptitle(short_name + ' Cutout Centroid Positions',
fontsize=title_size)
output_filename = pines_path / ('Objects/' + short_name +
'/analysis/diagnostic_plots/' +
short_name + '_cutout_positions.png')
plt.savefig(output_filename, dpi=300)
return
def epsf_phot(target, centroided_sources, plots=False):
def hmsm_to_days(hour=0,min=0,sec=0,micro=0):
"""
Convert hours, minutes, seconds, and microseconds to fractional days.
"""
days = sec + (micro / 1.e6)
days = min + (days / 60.)
days = hour + (days / 60.)
return days / 24.
def date_to_jd(year,month,day):
"""
Convert a date to Julian Day.
Algorithm from 'Practical Astronomy with your Calculator or Spreadsheet',
4th ed., Duffet-Smith and Zwart, 2011.
"""
if month == 1 or month == 2:
yearp = year - 1
monthp = month + 12
else:
yearp = year
monthp = month
# this checks where we are in relation to October 15, 1582, the beginning
# of the Gregorian calendar.
if ((year < 1582) or
(year == 1582 and month < 10) or
(year == 1582 and month == 10 and day < 15)):
# before start of Gregorian calendar
B = 0
else:
# after start of Gregorian calendar
A = math.trunc(yearp / 100.)
B = 2 - A + math.trunc(A / 4.)
if yearp < 0:
C = math.trunc((365.25 * yearp) - 0.75)
else:
C = math.trunc(365.25 * yearp)
D = math.trunc(30.6001 * (monthp + 1))
jd = B + C + D + day + 1720994.5
return jd
pines_path = pines_dir_check()
short_name = short_name_creator(target)
reduced_path = pines_path/('Objects/'+short_name+'/reduced/')
reduced_filenames = natsort.natsorted([x.name for x in reduced_path.glob('*.fits')])
reduced_files = np.array([reduced_path/i for i in reduced_filenames])
centroided_sources.columns = centroided_sources.columns.str.strip()
source_names = natsort.natsorted(list(set([i.split(' ')[0]+' '+i.split(' ')[1] for i in centroided_sources.keys() if (i[0] == '2') or (i[0] == 'R')])))
#Create output plot directories for each source.
if plots:
for name in source_names:
#If the folders are already there, delete them.
source_path = (pines_path/('Objects/'+short_name+'/psf_phot/'+name+'/'))
if source_path.exists():
shutil.rmtree(source_path)
#Create folders.
os.mkdir(source_path)
#Declare a new dataframe to hold the information for all targets for this .
columns = ['Filename', 'Time UT', 'Time JD', 'Airmass', 'Seeing']
for i in range(0, len(source_names)):
columns.append(source_names[i]+' Flux')
columns.append(source_names[i]+' Flux Error')
psf_df = pd.DataFrame(index=range(len(reduced_files)), columns=columns)
output_filename = pines_path/('Objects/'+short_name+'/psf_phot/'+short_name+'_psf_phot.csv')
for i in range(0, len(reduced_files)):
#Read in image data/header.
file = reduced_files[i]
data = fits.open(file)[0].data
header = fits.open(file)[0].header
print('{}, image {} of {}.'.format(file.name, i+1, len(reduced_files)))
#Read in some supporting information.
log_path = pines_path/('Logs/'+file.name.split('.')[0]+'_log.txt')
log = pines_log_reader(log_path)
date_obs = header['DATE-OBS']
#Catch a case that can cause datetime strptime to crash; Mimir headers sometimes have DATE-OBS with seconds specified as 010.xx seconds, when it should be 10.xx seconds.
if len(date_obs.split(':')[-1].split('.')[0]) == 3:
date_obs = date_obs.split(':')[0] + ':' + date_obs.split(':')[1] + ':' + date_obs.split(':')[-1][1:]
#Keep a try/except clause here in case other unknown DATE-OBS formats pop up.
try:
date = datetime.datetime.strptime(date_obs, '%Y-%m-%dT%H:%M:%S.%f')
except:
print('Header DATE-OBS format does not match the format code in strptime! Inspect/correct the DATE-OBS value.')
pdb.set_trace()
days = date.day + hmsm_to_days(date.hour,date.minute,date.second,date.microsecond)
jd = date_to_jd(date.year,date.month,days)
psf_df['Filename'][i] = file.name
psf_df['Time UT'][i] = header['DATE-OBS']
psf_df['Time JD'][i] = jd
psf_df['Airmass'][i] = header['AIRMASS']
psf_df['Seeing'][i] = log['X seeing'][np.where(log['Filename'] == file.name.split('_')[0]+'.fits')[0][0]]
#Read in source centroids for this image
x = np.zeros(len(source_names))
y = np.zeros(len(source_names))
for j in range(len(source_names)):
source = source_names[j]
x[j] = centroided_sources[source+' X'][i]
y[j] = centroided_sources[source+' Y'][i]
#Extract pixel cutouts of our stars, so let’s explicitly exclude stars that are too close to the image boundaries (because they cannot be extracted).
size = 13
hsize = (size - 1) / 2
#mask = ((x > hsize) & (x < (data.shape[1] -1 - hsize)) & (y > hsize) & (y < (data.shape[0] -1 - hsize)) & (y > 100) & (y < 923))
#Create table of good star positions
stars_tbl = Table()
stars_tbl['x'] = x
stars_tbl['y'] = y
#Subtract background (star cutouts from which we build the ePSF must have background subtracted).
mean_val, median_val, std_val = sigma_clipped_stats(data, sigma=2.)
data -= median_val
#Replace nans in data using Gaussian.
# kernel = Gaussian2DKernel(x_stddev=0.5)
# data = interpolate_replace_nans(data, kernel)
#The extract_stars() function requires the input data as an NDData object.
nddata = NDData(data=data)
#Extract star cutouts.
stars = extract_stars(nddata, stars_tbl, size=size)
#Plot.
nrows = 5
ncols = 5
fig, ax = plt.subplots(nrows=nrows, ncols=ncols, figsize=(10, 10), squeeze=True)
ax = ax.ravel()
for j in range(len(stars)):
norm = simple_norm(stars[j], 'log', percent=99.)
ax[j].imshow(stars[j].data, norm=norm, origin='lower', cmap='viridis')
pdb.set_trace()
#Construct the ePSF using the star cutouts.
epsf_fitter = EPSFFitter()
epsf_builder = EPSFBuilder(maxiters=4, progress_bar=False, fitter=epsf_fitter)
try:
epsf, fitted_stars = epsf_builder(stars)
output_filename = pines_path/('Objects/'+short_name+'/psf_phot/'+short_name+'_psf_phot.csv')
for j in range(len(stars)):
star = stars[j]
source_name = source_names[j]
sigma_psf = 1.85
dtype = [('x_0', 'f8'), ('y_0', 'f8')]
pos = Table(data=np.zeros(1, dtype=dtype))
source_x = stars_tbl['x'][j]
source_y = stars_tbl['y'][j]
pos['x_0'] = source_x - int(source_x - size/2 + 1)
pos['y_0'] = source_y - int(source_y - size/2 + 1)
daogroup = DAOGroup(4.0*sigma_psf*gaussian_sigma_to_fwhm)
mmm_bkg = MMMBackground()
photometry = BasicPSFPhotometry(group_maker=daogroup,
bkg_estimator=mmm_bkg,
psf_model=epsf,
fitter=LevMarLSQFitter(),
fitshape=(13,13),
aperture_radius=4.)
result_tab = photometry(image=star, init_guesses=pos)
residual_image = photometry.get_residual_image()
psf_df[source_name+' Flux'][i] = result_tab['flux_fit'][0]
psf_df[source_name+' Flux Error'][i] = result_tab['flux_unc'][0]
if plots:
fig, ax = plt.subplots(nrows=1, ncols=3, figsize=(12,4))
im = ax[0].imshow(star, origin='lower')
divider = make_axes_locatable(ax[0])
cax = divider.append_axes('right', size='5%', pad=0.05)
fig.colorbar(im, cax=cax, orientation='vertical')
ax[0].plot(result_tab['x_fit'][0], result_tab['y_fit'][0], 'rx')
ax[0].set_title('Data')
im2 = ax[1].imshow(epsf.data, origin='lower')
ax[1].set_title('EPSF Model')
divider = make_axes_locatable(ax[1])
cax = divider.append_axes('right', size='5%', pad=0.05)
fig.colorbar(im2, cax=cax, orientation='vertical')
im3 = ax[2].imshow(residual_image, origin='lower')
ax[2].set_title('Residual Image')
divider = make_axes_locatable(ax[2])
cax = divider.append_axes('right', size='5%', pad=0.05)
fig.colorbar(im3, cax=cax, orientation='vertical')
plt.suptitle(source_name+'\n'+reduced_files[i].name+', image '+str(i+1)+' of '+str(len(reduced_files)))
plt.subplots_adjust(wspace=0.5, top=0.95, bottom = 0.05)
plot_output_name = pines_path/('Objects/'+short_name+'/psf_phot/'+source_name+'/'+str(i).zfill(4)+'.jpg')
plt.savefig(plot_output_name)
plt.close()
except:
print('')
print('EPSF BUILDER FAILED, SKIPPING IMAGE.')
print('')
#Plot the ePSF.
# plt.figure()
# norm = simple_norm(epsf.data, 'log', percent=99.)
# plt.imshow(epsf.data, norm=norm, origin='lower', cmap='viridis')
# cb = plt.colorbar()
# plt.tight_layout()
print('Saving psf photometry output to {}.'.format(output_filename))
with open(output_filename, 'w') as f:
for j in range(len(psf_df)):
if j == 0:
f.write('{:>21s}, {:>22s}, {:>17s}, {:>7s}, {:>7s}, '.format('Filename', 'Time UT', 'Time JD', 'Airmass', 'Seeing'))
for i in range(len(source_names)):
if i != len(source_names) - 1:
f.write('{:>20s}, {:>26s}, '.format(source_names[i]+' Flux', source_names[i]+' Flux Error'))
else:
f.write('{:>20s}, {:>26s}\n'.format(source_names[i]+' Flux', source_names[i]+' Flux Error'))
format_string = '{:21s}, {:22s}, {:17.9f}, {:7.2f}, {:7.1f}, '
#If the seeing value for this image is 'nan' (a string), convert it to a float.
#TODO: Not sure why it's being read in as a string, fix that.
if type(psf_df['Seeing'][j]) == str:
psf_df['Seeing'][j] = float(psf_df['Seeing'][j])
#Do a try/except clause for writeout, in case it breaks in the future.
try:
f.write(format_string.format(psf_df['Filename'][j], psf_df['Time UT'][j], psf_df['Time JD'][j], psf_df['Airmass'][j], psf_df['Seeing'][j]))
except:
print('Writeout failed! Inspect quantities you are trying to write out.')
pdb.set_trace()
for i in range(len(source_names)):
if i != len(source_names) - 1:
format_string = '{:20.11f}, {:26.11f}, '
else:
format_string = '{:20.11f}, {:26.11f}\n'
f.write(format_string.format(psf_df[source_names[i]+' Flux'][j], psf_df[source_names[i]+' Flux Error'][j]))
print('')
return
def bpm_maker(flat_date, dark_date, exptime, band, upload=False, sftp=''):
pines_path = pines_dir_check()
#Load in the different masks.
kokopelli_path = pines_path / (
'Calibrations/Kokopelli Mask/kokopelli_mask.fits')
kokopelli_mask = (
1 - fits.open(kokopelli_path)[0].data).astype('int')[0:1024, :]
variable_path = pines_path / ('Calibrations/Variable Pixel Masks/vpm_' +
str(exptime) + '_s_' + dark_date + '.fits')
variable_mask = fits.open(variable_path)[0].data
hot_path = pines_path / ('Calibrations/Hot Pixel Masks/hpm_' +
str(exptime) + '_s_' + dark_date + '.fits')
hot_mask = fits.open(hot_path)[0].data
dead_path = pines_path / ('Calibrations/Dead Pixel Masks/dpm_' + band +
'_' + flat_date + '.fits')
dead_mask = fits.open(dead_path)[0].data
#Visualize all masks
bpm = np.zeros(np.shape(dead_mask), dtype='int')
bad_locs = np.where((kokopelli_mask == 1) | (variable_mask == 1)
| (hot_mask == 1) | (dead_mask == 1))
bpm[bad_locs] = 1
num_bad = len(np.where(bpm == 1)[0])
frac_bad = num_bad / 1024**2
print('{} percent of the detector flagged as bad.'.format(
np.round(frac_bad * 100, 1)))
# plt.ion()
# plt.imshow(bpm, origin='lower')
output_filename = 'bpm_' + band + '_' + str(
exptime) + '_s_' + flat_date + '.fits'
output_path = pines_path / ('Calibrations/Bad Pixel Masks/' +
output_filename)
hdu = fits.PrimaryHDU(bpm)
hdu.header['HIERARCH DATE CREATED'] = datetime.utcnow().strftime(
'%Y-%m-%d') + 'T' + datetime.utcnow().strftime('%H:%M:%S')
#Now save to a file on your local machine.
print('')
print('Writing the file to ' + output_filename)
#Check to see if other files of this name exist.
# if os.path.exists(output_path):
# print('')
# print('WARNING: This will overwrite {}!'.format(output_path))
# dark_check = input('Do you want to continue? y/n: ')
# if dark_check == 'y':
# hdu.writeto(output_path,overwrite=True)
# print('Wrote to {}!'.format(output_path))
# else:
# print('Not overwriting!')
# else:
hdu.writeto(output_path, overwrite=True)
print('Wrote to {}!'.format(output_path))
print('')
#Upload the master dark to PINES server.
if upload:
print('Beginning upload process to pines.bu.edu...')
print('Note, only PINES admins will be able to upload.')
print('')
sftp.chdir('/')
sftp.chdir('data/calibrations/Bad Pixel Masks')
upload_name = output_filename
# if upload_name in sftp.listdir():
# print('WARNING: This will overwrite {} in pines.bu.edu:data/calibrations/Bad Pixel Masks/'.format(upload_name))
# upload_check = input('Do you want to continue? y/n: ')
# if upload_check == 'y':
# sftp.put(output_path,upload_name)
# print('Uploaded to pines.bu.edu:data/calibrations/Bad Pixel Masks/!')
# else:
# print('Skipping upload!')
# else:
sftp.put(output_path, upload_name)
print(
'Uploaded {} to pines.bu.edu:data/calibrations/Bad Pixel Masks/!'.
format(upload_name))
def centroider(target,
sources,
output_plots=False,
gif=False,
restore=False,
box_w=8):
matplotlib.use('TkAgg')
plt.ioff()
t1 = time.time()
pines_path = pines_dir_check()
short_name = short_name_creator(target)
kernel = Gaussian2DKernel(x_stddev=1) #For fixing nans in cutouts.
#If restore == True, read in existing output and return.
if restore:
centroid_df = pd.read_csv(
pines_path / ('Objects/' + short_name +
'/sources/target_and_references_centroids.csv'),
converters={
'X Centroids': eval,
'Y Centroids': eval
})
print('Restoring centroider output from {}.'.format(
pines_path / ('Objects/' + short_name +
'/sources/target_and_references_centroids.csv')))
print('')
return centroid_df
#Create subdirectories in sources folder to contain output plots.
if output_plots:
subdirs = glob(
str(pines_path / ('Objects/' + short_name + '/sources')) + '/*/')
#Delete any source directories that are already there.
for name in subdirs:
shutil.rmtree(name)
#Create new source directories.
for name in sources['Name']:
source_path = (
pines_path /
('Objects/' + short_name + '/sources/' + name + '/'))
os.mkdir(source_path)
#Read in extra shifts, in case the master image wasn't used for source detection.
extra_shift_path = pines_path / ('Objects/' + short_name +
'/sources/extra_shifts.txt')
extra_shifts = pd.read_csv(extra_shift_path,
delimiter=' ',
names=['Extra X shift', 'Extra Y shift'])
extra_x_shift = extra_shifts['Extra X shift'][0]
extra_y_shift = extra_shifts['Extra Y shift'][0]
np.seterr(
divide='ignore', invalid='ignore'
) #Suppress some warnings we don't care about in median combining.
#Get list of reduced files for target.
reduced_path = pines_path / ('Objects/' + short_name + '/reduced')
reduced_filenames = natsort.natsorted(
[x.name for x in reduced_path.glob('*red.fits')])
reduced_files = np.array([reduced_path / i for i in reduced_filenames])
#Declare a new dataframe to hold the centroid information for all sources we want to track.
columns = []
columns.append('Filename')
columns.append('Seeing')
columns.append('Time (JD UTC)')
columns.append('Airmass')
#Add x/y positions and cenroid flags for every tracked source
for i in range(0, len(sources)):
columns.append(sources['Name'][i] + ' Image X')
columns.append(sources['Name'][i] + ' Image Y')
columns.append(sources['Name'][i] + ' Cutout X')
columns.append(sources['Name'][i] + ' Cutout Y')
columns.append(sources['Name'][i] + ' Centroid Warning')
centroid_df = pd.DataFrame(index=range(len(reduced_files)),
columns=columns)
log_path = pines_path / ('Logs/')
log_dates = np.array(
natsort.natsorted(
[x.name.split('_')[0] for x in log_path.glob('*.txt')]))
#Make sure we have logs for all the nights of these data. Need them to account for image shifts.
nights = list(set([i.name.split('.')[0] for i in reduced_files]))
for i in nights:
if i not in log_dates:
print('ERROR: {} not in {}. Download it from the PINES server.'.
format(i + '_log.txt', log_path))
pdb.set_trace()
shift_tolerance = 2.0 #Number of pixels that the measured centroid can be away from the expected position in either x or y before trying other centroiding algorithms.
for i in range(len(sources)):
#Get the initial source position.
x_pos = sources['Source Detect X'][i]
y_pos = sources['Source Detect Y'][i]
print('')
print(
'Getting centroids for {}, ({:3.1f}, {:3.1f}) in source detection image. Source {} of {}.'
.format(sources['Name'][i], x_pos, y_pos, i + 1, len(sources)))
if output_plots:
print('Saving centroid plots to {}.'.format(
pines_path / ('Objects/' + short_name + '/sources/' +
sources['Name'][i] + '/')))
pbar = ProgressBar()
for j in pbar(range(len(reduced_files))):
centroid_df[sources['Name'][i] + ' Centroid Warning'][j] = 0
file = reduced_files[j]
image = fits.open(file)[0].data
#Get the measured image shift for this image.
log = pines_log_reader(log_path /
(file.name.split('.')[0] + '_log.txt'))
log_ind = np.where(log['Filename'] == file.name.split('_')[0] +
'.fits')[0][0]
x_shift = float(log['X shift'][log_ind])
y_shift = float(log['Y shift'][log_ind])
#Save the filename for readability. Save the seeing for use in variable aperture photometry. Save the time for diagnostic plots.
if i == 0:
centroid_df['Filename'][j] = file.name.split('_')[0] + '.fits'
centroid_df['Seeing'][j] = log['X seeing'][log_ind]
time_str = fits.open(file)[0].header['DATE-OBS']
#Correct some formatting issues that can occur in Mimir time stamps.
if time_str.split(':')[-1] == '60.00':
time_str = time_str[0:14] + str(
int(time_str.split(':')[-2]) + 1) + ':00.00'
elif time_str.split(':')[-1] == '010.00':
time_str = time_str[0:17] + time_str.split(':')[-1][1:]
centroid_df['Time (JD UTC)'][j] = julian.to_jd(
datetime.datetime.strptime(time_str,
'%Y-%m-%dT%H:%M:%S.%f'))
centroid_df['Airmass'][j] = log['Airmass'][log_ind]
nan_flag = False #Flag indicating if you should not trust the log's shifts. Set to true if x_shift/y_shift are 'nan' or > 30 pixels.
#If bad shifts were measured for this image, skip.
if log['Shift quality flag'][log_ind] == 1:
continue
if np.isnan(x_shift) or np.isnan(y_shift):
x_shift = 0
y_shift = 0
nan_flag = True
#If there are clouds, shifts could have been erroneously high...just zero them?
if abs(x_shift) > 200:
#x_shift = 0
nan_flag = True
if abs(y_shift) > 200:
#y_shift = 0
nan_flag = True
#Apply the shift. NOTE: This relies on having accurate x_shift and y_shift values from the log.
#If they're incorrect, the cutout will not be in the right place.
#x_pos = sources['Source Detect X'][i] - x_shift + extra_x_shift
#y_pos = sources['Source Detect Y'][i] + y_shift - extra_y_shift
x_pos = sources['Source Detect X'][i] - (x_shift - extra_x_shift)
y_pos = sources['Source Detect Y'][i] + (y_shift - extra_y_shift)
#TODO: Make all this its own function.
#Cutout around the expected position and interpolate over any NaNs (which screw up source detection).
cutout = interpolate_replace_nans(
image[int(y_pos - box_w):int(y_pos + box_w) + 1,
int(x_pos - box_w):int(x_pos + box_w) + 1],
kernel=Gaussian2DKernel(x_stddev=0.5))
#interpolate_replace_nans struggles with edge pixels, so shave off edge_shave pixels in each direction of the cutout.
edge_shave = 1
cutout = cutout[edge_shave:len(cutout) - edge_shave,
edge_shave:len(cutout) - edge_shave]
vals, lower, upper = sigmaclip(
cutout, low=1.5,
high=2.5) #Get sigma clipped stats on the cutout
med = np.nanmedian(vals)
std = np.nanstd(vals)
try:
centroid_x_cutout, centroid_y_cutout = centroid_2dg(
cutout - med) #Perform centroid detection on the cutout.
except:
pdb.set_trace()
centroid_x = centroid_x_cutout + int(
x_pos
) - box_w + edge_shave #Translate the detected centroid from the cutout coordinates back to the full-frame coordinates.
centroid_y = centroid_y_cutout + int(y_pos) - box_w + edge_shave
# if i == 0:
# qp(cutout)
# plt.plot(centroid_x_cutout, centroid_y_cutout, 'rx')
# # qp(image)
# # plt.plot(centroid_x, centroid_y, 'rx')
# pdb.set_trace()
#If the shifts in the log are not 'nan' or > 200 pixels, check if the measured shifts are within shift_tolerance pixels of the expected position.
# If they aren't, try alternate centroiding methods to try and find it.
#Otherwise, use the shifts as measured with centroid_1dg. PINES_watchdog likely failed while observing, and we don't expect the centroids measured here to actually be at the expected position.
if not nan_flag:
#Try a 2D Gaussian detection.
if (abs(centroid_x - x_pos) > shift_tolerance) or (
abs(centroid_y - y_pos) > shift_tolerance):
centroid_x_cutout, centroid_y_cutout = centroid_2dg(
cutout - med)
centroid_x = centroid_x_cutout + int(x_pos) - box_w
centroid_y = centroid_y_cutout + int(y_pos) - box_w
#If that fails, try a COM detection.
if (abs(centroid_x - x_pos) > shift_tolerance) or (
abs(centroid_y - y_pos) > shift_tolerance):
centroid_x_cutout, centroid_y_cutout = centroid_com(
cutout - med)
centroid_x = centroid_x_cutout + int(x_pos) - box_w
centroid_y = centroid_y_cutout + int(y_pos) - box_w
#If that fails, try masking source and interpolate over any bad pixels that aren't in the bad pixel mask, then redo 1D gaussian detection.
if (abs(centroid_x - x_pos) > shift_tolerance) or (
abs(centroid_y - y_pos) > shift_tolerance):
mask = make_source_mask(cutout,
nsigma=4,
npixels=5,
dilate_size=3)
vals, lo, hi = sigmaclip(cutout[~mask])
bad_locs = np.where((mask == False) & (
(cutout > hi) | (cutout < lo)))
cutout[bad_locs] = np.nan
cutout = interpolate_replace_nans(
cutout, kernel=Gaussian2DKernel(x_stddev=0.5))
centroid_x_cutout, centroid_y_cutout = centroid_1dg(
cutout - med)
centroid_x = centroid_x_cutout + int(x_pos) - box_w
centroid_y = centroid_y_cutout + int(y_pos) - box_w
#Try a 2D Gaussian detection on the interpolated cutout
if (abs(centroid_x - x_pos) > shift_tolerance) or (
abs(centroid_y - y_pos) > shift_tolerance):
centroid_x_cutout, centroid_y_cutout = centroid_2dg(
cutout - med)
centroid_x = centroid_x_cutout + int(
x_pos) - box_w
centroid_y = centroid_y_cutout + int(
y_pos) - box_w
#Try a COM on the interpolated cutout.
if (abs(centroid_x - x_pos) > shift_tolerance
) or (abs(centroid_y - y_pos) >
shift_tolerance):
centroid_x_cutout, centroid_y_cutout = centroid_com(
cutout)
centroid_x = centroid_x_cutout + int(
x_pos) - box_w
centroid_y = centroid_y_cutout + int(
y_pos) - box_w
#Last resort: try cutting off the edge of the cutout. Edge pixels can experience poor interpolation, and this sometimes helps.
if (abs(centroid_x - x_pos) >
shift_tolerance) or (
abs(centroid_y - y_pos) >
shift_tolerance):
cutout = cutout[1:-1, 1:-1]
centroid_x_cutout, centroid_y_cutout = centroid_1dg(
cutout - med)
centroid_x = centroid_x_cutout + int(
x_pos) - box_w + 1
centroid_y = centroid_y_cutout + int(
y_pos) - box_w + 1
#Try with a 2DG
if (abs(centroid_x - x_pos) >
shift_tolerance) or (
abs(centroid_y - y_pos) >
shift_tolerance):
centroid_x_cutout, centroid_y_cutout = centroid_2dg(
cutout - med)
centroid_x = centroid_x_cutout + int(
x_pos) - box_w + 1
centroid_y = centroid_y_cutout + int(
y_pos) - box_w + 1
#If ALL that fails, report the expected position as the centroid.
if (abs(centroid_x - x_pos) >
shift_tolerance) or (
abs(centroid_y - y_pos)
> shift_tolerance):
print(
'WARNING: large centroid deviation measured, returning predicted position'
)
print('')
centroid_df[
sources['Name'][i] +
' Centroid Warning'][j] = 1
centroid_x = x_pos
centroid_y = y_pos
#pdb.set_trace()
#Check that your measured position is actually on the detector.
if (centroid_x < 0) or (centroid_y < 0) or (centroid_x > 1023) or (
centroid_y > 1023):
#Try a quick mask/interpolation of the cutout.
mask = make_source_mask(cutout,
nsigma=3,
npixels=5,
dilate_size=3)
vals, lo, hi = sigmaclip(cutout[~mask])
bad_locs = np.where((mask == False)
& ((cutout > hi) | (cutout < lo)))
cutout[bad_locs] = np.nan
cutout = interpolate_replace_nans(
cutout, kernel=Gaussian2DKernel(x_stddev=0.5))
centroid_x, centroid_y = centroid_2dg(cutout - med)
centroid_x += int(x_pos) - box_w
centroid_y += int(y_pos) - box_w
if (centroid_x < 0) or (centroid_y < 0) or (
centroid_x > 1023) or (centroid_y > 1023):
print(
'WARNING: large centroid deviation measured, returning predicted position'
)
print('')
centroid_df[sources['Name'][i] +
' Centroid Warning'][j] = 1
centroid_x = x_pos
centroid_y = y_pos
#pdb.set_trace()
#Check to make sure you didn't measure nan's.
if np.isnan(centroid_x):
centroid_x = x_pos
print(
'NaN returned from centroid algorithm, defaulting to target position in source_detct_image.'
)
if np.isnan(centroid_y):
centroid_y = y_pos
print(
'NaN returned from centroid algorithm, defaulting to target position in source_detct_image.'
)
#Record the image and relative cutout positions.
centroid_df[sources['Name'][i] + ' Image X'][j] = centroid_x
centroid_df[sources['Name'][i] + ' Image Y'][j] = centroid_y
centroid_df[sources['Name'][i] +
' Cutout X'][j] = centroid_x_cutout
centroid_df[sources['Name'][i] +
' Cutout Y'][j] = centroid_y_cutout
if output_plots:
#Plot
lock_x = int(centroid_df[sources['Name'][i] + ' Image X'][0])
lock_y = int(centroid_df[sources['Name'][i] + ' Image Y'][0])
norm = ImageNormalize(data=cutout, interval=ZScaleInterval())
plt.imshow(image, origin='lower', norm=norm)
plt.plot(centroid_x, centroid_y, 'rx')
ap = CircularAperture((centroid_x, centroid_y), r=5)
ap.plot(lw=2, color='b')
plt.ylim(lock_y - 30, lock_y + 30 - 1)
plt.xlim(lock_x - 30, lock_x + 30 - 1)
plt.title('CENTROID DIAGNOSTIC PLOT\n' + sources['Name'][i] +
', ' + reduced_files[j].name + ' (image ' +
str(j + 1) + ' of ' + str(len(reduced_files)) + ')',
fontsize=10)
plt.text(centroid_x,
centroid_y + 0.5,
'(' + str(np.round(centroid_x, 1)) + ', ' +
str(np.round(centroid_y, 1)) + ')',
color='r',
ha='center')
plot_output_path = (
pines_path /
('Objects/' + short_name + '/sources/' +
sources['Name'][i] + '/' + str(j).zfill(4) + '.jpg'))
plt.gca().set_axis_off()
plt.subplots_adjust(top=1,
bottom=0,
right=1,
left=0,
hspace=0,
wspace=0)
plt.margins(0, 0)
plt.gca().xaxis.set_major_locator(plt.NullLocator())
plt.gca().yaxis.set_major_locator(plt.NullLocator())
plt.savefig(plot_output_path,
bbox_inches='tight',
pad_inches=0,
dpi=150)
plt.close()
if gif:
gif_path = (pines_path / ('Objects/' + short_name + '/sources/' +
sources['Name'][i] + '/'))
gif_maker(path=gif_path, fps=10)
output_filename = pines_path / (
'Objects/' + short_name +
'/sources/target_and_references_centroids.csv')
#centroid_df.to_csv(pines_path/('Objects/'+short_name+'/sources/target_and_references_centroids.csv'))
print('Saving centroiding output to {}.'.format(output_filename))
with open(output_filename, 'w') as f:
for j in range(len(centroid_df)):
#Write the header line.
if j == 0:
f.write('{:<17s}, '.format('Filename'))
f.write('{:<15s}, '.format('Time (JD UTC)'))
f.write('{:<6s}, '.format('Seeing'))
f.write('{:<7s}, '.format('Airmass'))
for i in range(len(sources['Name'])):
n = sources['Name'][i]
if i != len(sources['Name']) - 1:
f.write(
'{:<23s}, {:<23s}, {:<24s}, {:<24s}, {:<34s}, '.
format(n + ' Image X', n + ' Image Y',
n + ' Cutout X', n + ' Cutout Y',
n + ' Centroid Warning'))
else:
f.write(
'{:<23s}, {:<23s}, {:<24s}, {:<24s}, {:<34s}\n'.
format(n + ' Image X', n + ' Image Y',
n + ' Cutout X', n + ' Cutout Y',
n + ' Centroid Warning'))
#Write in the data lines.
try:
f.write('{:<17s}, '.format(centroid_df['Filename'][j]))
f.write('{:<15.7f}, '.format(centroid_df['Time (JD UTC)'][j]))
f.write('{:<6.1f}, '.format(float(centroid_df['Seeing'][j])))
f.write('{:<7.2f}, '.format(centroid_df['Airmass'][j]))
except:
pdb.set_trace()
for i in range(len(sources['Name'])):
n = sources['Name'][i]
if i != len(sources['Name']) - 1:
format_string = '{:<23.4f}, {:<23.4f}, {:<24.4f}, {:<24.4f}, {:<34d}, '
else:
format_string = '{:<23.4f}, {:<23.4f}, {:<24.4f}, {:<24.4f}, {:<34d}\n'
f.write(
format_string.format(
centroid_df[n + ' Image X'][j],
centroid_df[n + ' Image Y'][j],
centroid_df[n + ' Cutout X'][j],
centroid_df[n + ' Cutout Y'][j],
centroid_df[n + ' Centroid Warning'][j]))
np.seterr(divide='warn', invalid='warn')
print('')
print('centroider runtime: {:.2f} minutes.'.format(
(time.time() - t1) / 60))
print('')
return centroid_df
def movie(long_target_name,
dates_to_examine=[],
target_number=0,
box_size=300,
show_centroid=1):
import time
'''Authors:
Patrick Tamburo, Boston University, Jan 2020, July 2020
Purpose:
Inputs:
Outputs:
TODO:
'''
warnings.filterwarnings(
'ignore', category=UserWarning, append=True
) #This turns of NaN warnings in sigma_clipped_stats, otherwise we'd get a warning every line.
kernel = Gaussian2DKernel(x_stddev=0.5)
pines_path = pines_dir_check()
target_name = short_name_creator(long_target_name)
object_path = pines_path / ('Objects/' + target_name + '/')
reduced_path = object_path / 'reduced/'
reduced_files = np.array(
natsort.natsorted([x for x in reduced_path.glob('*.fits')]))
centroid_path = object_path / 'sources/'
positions = pd.read_csv(centroid_path /
'target_and_references_centroids.csv')
x_positions = np.array(positions[positions.keys()[2 * target_number]])
y_positions = np.array(positions[positions.keys()[2 * target_number + 1]])
initial_position = (int(x_positions[0]), int(y_positions[0]))
#Get list of files
all_frame_list = np.array([])
for i in range(np.size(reduced_files)):
file_name = reduced_files[i].name
all_frame_list = np.append(all_frame_list, file_name)
num_files = len(reduced_files)
dates = np.array([
reduced_files[i].name.split('.')[0] for i in range(len(reduced_files))
])
plt.ion()
fig, ax = plt.subplots(1, 1, figsize=(8, 7))
for i in range(len(reduced_files)):
if len(dates_to_examine) != 0:
if reduced_files[i].name.split('.')[0] in dates_to_examine:
image_path = reduced_files[i]
title = image_path.name
image = fits.open(image_path)[0].data
header = fits.open(image_path)[0].header
image = interpolate_replace_nans(image, kernel)
image_bg = Background2D(image, 64)
image = image - image_bg.background
frame = image[initial_position[1] -
int(box_size / 2):initial_position[1] +
int(box_size / 2), initial_position[0] -
int(box_size / 2):initial_position[0] +
int(box_size / 2)]
norm = ImageNormalize(frame,
interval=ZScaleInterval(),
stretch=SquaredStretch())
im = ax.imshow(image, origin='lower', norm=norm)
ax.set_xlim(initial_position[0] - int(box_size / 2),
initial_position[0] + int(box_size / 2))
ax.set_ylim(initial_position[1] - int(box_size / 2),
initial_position[1] + int(box_size / 2))
if show_centroid:
ax.plot(x_positions[i], y_positions[i], 'mo')
ax.set_title(title)
plt.pause(0.01)
ax.cla()
else:
image_path = reduced_files[i]
title = image_path.name
image = fits.open(image_path)[0].data
header = fits.open(image_path)[0].header
frame = image[initial_position[1] -
int(box_size / 2):initial_position[1] +
int(box_size / 2), initial_position[0] -
int(box_size / 2):initial_position[0] +
int(box_size / 2)]
avg, med, std = sigma_clipped_stats(image)
im = ax.imshow(image, origin='lower', vmin=med, vmax=med + 5 * std)
ax.set_xlim(initial_position[0] - int(box_size / 2),
initial_position[0] + int(box_size / 2))
ax.set_ylim(initial_position[1] - int(box_size / 2),
initial_position[1] + int(box_size / 2))
cb = fig.colorbar(im, orientation='vertical', label='Counts')
if show_centroid:
ax.plot(x_positions[i], y_positions[i], 'bx')
ax.set_title(title)
plt.pause(0.01)
ax.cla()
cb.remove()
def dark(date, exptime, dark_start=0, dark_stop=0, upload=False, delete_raw=False, sftp=''):
clip_lvl = 3 #The value to use for sigma clipping.
pines_path = pines_dir_check()
np.seterr(invalid='ignore') #Suppress some warnings we don't care about in median combining.
exptime = float(exptime)
plt.ion() #Turn on interactive plotting.
t1 = time.time()
#If an sftp connection to the PINES server was passed, download the dark data.
if type(sftp) == pysftp.Connection:
sftp.chdir('/data/raw/mimir')
run_list = sftp.listdir()
data_path = '' #Initialize to check that it gets filled.
for i in range(len(run_list)):
sftp.chdir(run_list[i])
date_list = sftp.listdir()
if date in date_list:
data_path = sftp.getcwd()
print('{} directory found in pines.bu.edu:{}/\n'.format(date,data_path))
sftp.chdir(date)
break
sftp.chdir('..')
if data_path == '':
print('ERROR: specified date not found in any run on pines.bu.edu:data/raw/mimir/\n')
return
else:
#If the file start/stop numbers are specified, grab those files.
if (dark_stop != 0):
files_in_dir = sftp.listdir()
dark_filenums = np.arange(dark_start, dark_stop+1, step=1)
dark_files = []
#Add the darks to the file list.
for i in range(len(dark_filenums)):
file_num = dark_filenums[i]
#Generate the filename.
if file_num < 10:
file_name = date+'.00'+str(file_num)+'.fits'
elif (file_num >= 10) and (file_num < 100):
file_name = date+'.0'+str(file_num)+'.fits'
else:
file_name = date+'.'+str(file_num)+'.fits'
#Check if the file name is in the directory, and if so, append it to the list of flat files.
if file_name in files_in_dir:
dark_files.append(file_name)
else:
print('{} not found in directory, skipping.'.format(file_name))
pdb.set_trace()
else:
#Otherwise, find the files automatically using the night's log.
log_path = pines_path/'Logs'
#Check if you already have the log for this date, if not, download it.
#Download from the /data/logs/ directory on PINES.
if not (log_path/(date+'_log.txt')).exists():
print('Downloading {}_log.txt to {}\n'.format(date,log_path))
sftp.get('/data/logs/'+date+'_log.txt',log_path/(date+'_log.txt'))
#Read in the log from this date.
log = pines_log_reader(log_path/(date+'_log.txt'))
#Identify dark files.
dark_inds = np.where((log['Target'] == 'Dark') & (log['Filename'] != 'test.fits') & (log['Exptime'] == exptime))[0]
dark_files = natsort.natsorted(list(set(log['Filename'][dark_inds]))) #Set guarantees we only grab the unique files that have been identified as flats, in case the log bugged out.
print('Found {} dark files.'.format(len(dark_files)))
print('')
#Downoad data to the Calibrations/Darks/Raw/ directory.
dark_path = pines_path/('Calibrations/Darks/Raw')
for j in range(len(dark_files)):
if not (dark_path/dark_files[j]).exists():
sftp.get(dark_files[j],os.path.join(pines_path,dark_path/dark_files[j]))
print('Downloading {} to {}, {} of {}.'.format(dark_files[j], dark_path, j+1, len(dark_files)))
else:
print('{} already in {}, skipping download.'.format(dark_files[j],dark_path))
print('')
#If no sftp was passed, search for files on disk.
else:
dark_path = pines_path/('Calibrations/Darks/Raw')
all_dark_files = natsort.natsorted(list(Path(dark_path).rglob(date+'*.fits')))
dark_files = []
for file in all_dark_files:
if fits.open(file)[0].header['EXPTIME'] == exptime:
dark_files.append(file)
num_images = len(dark_files)
if num_images == 0:
raise RuntimeError('No raw dark files found on disk with date '+date+'!')
print('Reading in ', num_images,' dark images.')
dark_cube_raw = np.zeros([len(dark_files),1024,1024])
print('')
print('Dark frame information')
print('-------------------------------------------------')
print('ID Mean Stddev Max Min')
print('-------------------------------------------------')
for j in range(len(dark_files)):
image_data = fits.open(dark_path/dark_files[j])[0].data[0:1024,:] #This line trims off the top two rows of the image, which are overscan.
header = fits.open(dark_path/dark_files[j])[0].header
if header['EXPTIME'] != exptime:
print('ERROR: {} taken has exposure time different than than exptime.'.format(dark_files[j]))
return
dark_cube_raw[j,:,:] = image_data
print(str(j+1)+' '+str(np.mean(image_data))+' '+str(np.std(image_data))+' '+ str(np.amax(image_data))+' '+str(np.amin(image_data)))
cube_shape = np.shape(dark_cube_raw)
master_dark = np.zeros((cube_shape[1], cube_shape[2]), dtype='float32')
master_dark_stddev = np.zeros((cube_shape[1], cube_shape[2]), dtype='float32')
print('')
print('Combining the darks')
print('......')
pbar = ProgressBar()
#For each pixel, calculate the mean, median, and standard deviation "through the stack" of darks.
for x in pbar(range(cube_shape[1])):
for y in range(cube_shape[2]):
through_stack = dark_cube_raw[:,y,x]
through_stack_median = np.nanmedian(through_stack)
through_stack_stddev = np.nanstd(through_stack)
#Flag values that are > clip_lvl-sigma discrepant from the median.
good_inds = np.where((abs(through_stack - through_stack_median) / through_stack_stddev <= clip_lvl))[0]
#Calculate the sigma-clipped mean and sigma-clipped stddev using good_inds.
s_c_mean = np.nanmean(through_stack[good_inds])
s_c_stddev = np.nanstd(through_stack[good_inds])
#Store the sigma-clipped mean as the master dark value for this pixel.
master_dark[y,x] = s_c_mean
master_dark_stddev[y,x] = s_c_stddev
np.seterr(invalid='warn') #Turn invalid warnings back on, in case it would permanently turn it off otherwise.
output_filename = pines_path/('Calibrations/Darks/Master Darks/master_dark_'+str(exptime)+'_s_'+date+'.fits')
#Add some header keywords detailing the master_dark creation process.
hdu = fits.PrimaryHDU(master_dark)
hdu.header['HIERARCH DATE CREATED'] = datetime.utcnow().strftime('%Y-%m-%d')+'T'+datetime.utcnow().strftime('%H:%M:%S')
#Now save to a file on your local machine.
#Check to see if other files of this name exist.
# if os.path.exists(output_filename):
# print('')
# print('WARNING: This will overwrite {}!'.format(output_filename))
# dark_check = input('Do you want to continue? y/n: ')
# if dark_check == 'y':
# hdu.writeto(output_filename,overwrite=True)
# print('Wrote to {}!'.format(output_filename))
# else:
# print('Not overwriting!')
# else:
hdu.writeto(output_filename,overwrite=True)
print('Wrote to {}!'.format(output_filename))
#Upload the master dark to PINES server.
if upload:
print('Uploading to pines.bu.edu...')
sftp.chdir('..')
sftp.chdir('..')
sftp.chdir('..')
sftp.chdir('..')
sftp.chdir('calibrations/Darks')
upload_name = 'master_dark_'+str(exptime)+'_s_'+date+'.fits'
# if upload_name in sftp.listdir():
# print('WARNING: This will overwrite {} in pines.bu.edu:data/calibrations/Darks/'.format(upload_name))
# upload_check = input('Do you want to continue? y/n: ')
# if upload_check == 'y':
# sftp.put(output_filename,upload_name)
# print('Uploaded to pines.bu.edu:data/calibrations/Darks/!')
# else:
# print('Skipping upload!')
# else:
sftp.put(output_filename,upload_name)
print('Uploaded {} to pines.bu.edu:data/calibrations/Darks/!'.format(upload_name))
sftp.chdir('..')
#Do the same thing for the sigma-clipped standard deviation image.
output_filename = pines_path/('Calibrations/Darks/Master Darks Stddev/master_dark_stddev_'+str(exptime)+'_s_'+date+'.fits')
#Add some header keywords detailing the master_dark creation process.
hdu = fits.PrimaryHDU(master_dark_stddev)
hdu.header['HIERARCH DATE CREATED'] = datetime.utcnow().strftime('%Y-%m-%d')+'T'+datetime.utcnow().strftime('%H:%M:%S')
username = ''
#Now save to a file on your local machine.
print('')
#Check to see if other files of this name exist.
# if os.path.exists(output_filename):
# print('')
# print('WARNING: This will overwrite {}!'.format(output_filename))
# dark_check = input('Do you want to continue? y/n: ')
# if dark_check == 'y':
# hdu.writeto(output_filename,overwrite=True)
# print('Wrote to {}!'.format(output_filename))
# else:
# print('Not overwriting!')
# else:
hdu.writeto(output_filename,overwrite=True)
print('Wrote to {}!'.format(output_filename))
#Upload the master dark to PINES server.
if upload:
print('Uploading to pines.bu.edu...')
sftp.chdir('Darks Stddev')
upload_name = 'master_dark_stddev_'+str(exptime)+'_s_'+date+'.fits'
# if upload_name in sftp.listdir():
# print('WARNING: This will overwrite {} in pines.bu.edu:data/calibrations/Darks Stddev/'.format(upload_name))
# upload_check = input('Do you want to continue? y/n: ')
# if upload_check == 'y':
# sftp.put(output_filename,upload_name)
# print('Uploaded to pines.bu.edu:data/calibrations/Darks Stddev/!')
# else:
# print('Skipping upload!')
# else:
sftp.put(output_filename,upload_name)
print('Uploaded {} to pines.bu.edu:data/calibrations/Darks Stddev/!'.format(upload_name))
print('')
#Delete raw dark images from disk.
if delete_raw:
files_to_delete = glob.glob(os.path.join(dark_path/'*.fits'))
for j in range(len(files_to_delete)):
os.remove(files_to_delete[j])
print('dark runtime: ', np.round((time.time()-t1)/60,1), ' minutes.')
print('Done!')
def background_plot(target, centroided_sources, gain=8.21):
pines_path = pines_dir_check()
short_name = short_name_creator(target)
#Get plot style parameters.
title_size, axis_title_size, axis_ticks_font_size, legend_font_size = plot_style(
)
analysis_path = pines_path / ('Objects/' + short_name + '/analysis')
phot_path = pines_path / ('Objects/' + short_name + '/aper_phot')
phot_files = np.array(natsorted([x for x in phot_path.glob('*.csv')]))
if os.path.exists(analysis_path / ('optimal_aperture.txt')):
with open(analysis_path / ('optimal_aperture.txt'), 'r') as f:
best_ap = f.readlines()[0].split(': ')[1].split('_')[0]
ap_list = np.array(
[str(i).split('/')[-1].split('_')[4] for i in phot_files])
best_ap_ind = np.where(ap_list == best_ap)[0][0]
else:
print(
'No optimal_aperture.txt file for {}.\nUsing first photometry file in {}.'
.format(target, phot_path))
best_ap_ind = 0
phot_file = phot_files[best_ap_ind]
phot_df = pines_log_reader(phot_file)
backgrounds = np.array(phot_df[short_name + ' Background'],
dtype='float') / gain
times_full = np.array(phot_df['Time JD'], dtype='float')
night_inds = night_splitter(times_full)
num_nights = len(night_inds)
times_nights = [times_full[night_inds[i]] for i in range(num_nights)]
standard_x = standard_x_range(times_nights)
fig, ax = plt.subplots(nrows=1,
ncols=num_nights,
figsize=(17, 5),
sharey=True)
for i in range(num_nights):
if i == 0:
ax[i].set_ylabel('Background (ADU)', fontsize=axis_title_size)
inds = night_inds[i]
ax[i].plot(times_full[inds],
backgrounds[inds],
marker='.',
linestyle='',
color='tab:orange',
alpha=0.3,
label='Raw bkg.')
ax[i].tick_params(labelsize=axis_ticks_font_size)
ax[i].set_xlabel('Time (JD UTC)', fontsize=axis_title_size)
ax[i].grid(alpha=0.2)
ax[i].set_xlim(
np.mean(times_full[inds]) - standard_x / 2,
np.mean(times_full[inds] + standard_x / 2))
#bin
block_inds = block_splitter(times_full[inds])
block_x = np.zeros(len(block_inds))
block_y = np.zeros(len(block_inds))
block_y_err = np.zeros(len(block_inds))
for j in range(len(block_inds)):
block_x[j] = np.nanmean(times_full[inds][block_inds[j]])
block_y[j] = np.nanmean(backgrounds[inds][block_inds[j]])
block_y_err[j] = np.nanstd(
backgrounds[inds][block_inds[j]]) / np.sqrt(
len(backgrounds[inds][block_inds[j]]))
block_x = block_x[~np.isnan(block_y)]
block_y_err = block_y_err[~np.isnan(block_y)]
block_y = block_y[~np.isnan(block_y)]
ax[i].errorbar(block_x,
block_y,
block_y_err,
marker='o',
linestyle='',
color='tab:orange',
ms=8,
mfc='none',
mew=2,
label='Bin bkg.')
#Interpolate each night's seeing.
fit_times = np.linspace(block_x[0], block_x[-1], 1000)
try:
interp = CubicSpline(block_x, block_y)
except:
pdb.set_trace()
interp_fit = interp(fit_times)
ax[i].plot(fit_times,
interp_fit,
color='b',
lw=2,
zorder=0,
alpha=0.7,
label='CS Interp.')
ax[i].legend(bbox_to_anchor=(1.01, 0.5), fontsize=legend_font_size)
plt.suptitle(short_name + ' Background Measurements', fontsize=title_size)
plt.subplots_adjust(left=0.07, wspace=0.05, top=0.92, bottom=0.17)
output_filename = pines_path / ('Objects/' + short_name +
'/analysis/diagnostic_plots/' +
short_name + '_backgrounds.png')
plt.savefig(output_filename, dpi=300)
return
def fixed_aper_phot(target,
centroided_sources,
ap_radii,
an_in=12.,
an_out=30.,
plots=False,
gain=8.21,
qe=0.9):
'''Authors:
Patrick Tamburo, Boston University, June 2020
Purpose:
Performs *fixed* aperture photometry on a set of reduced images given dataframe of source positions.
The iraf_style_photometry, compute_phot_error, perture_stats_tbl, and calc_aperture_mmm routines are from Varun Bajaj on github:
https://github.com/spacetelescope/wfc3_photometry/blob/master/photometry_tools/photometry_with_errors.py.
Inputs:
target (str): The target's full 2MASS name.
sources (pandas dataframe): List of source names, x and y positions in every image.
ap_radii (list of floats): List of aperture radii in pixels for which aperture photometry wil be performed.
an_in (float, optional): The inner radius of the annulus used to estimate background, in pixels.
an_out (float, optional): The outer radius of the annulus used to estimate background, in pixels.
plots (bool, optional): Whether or not to output surface plots. Images output to aper_phot directory within the object directory.
gain (float, optional): The gain of the detector in e-/ADU.
qe (float, optional): The quantum efficiency of the detector.
Outputs:
Saves aperture photometry csv to PINES_analysis_toolkit/Objects/short_name/aper_phot/ for each aperture.
TODO:
'''
pines_path = pines_dir_check()
short_name = short_name_creator(target)
#Remove any leading/trailing spaces in the column names.
centroided_sources.columns = centroided_sources.columns.str.lstrip()
centroided_sources.columns = centroided_sources.columns.str.rstrip()
#Get list of reduced files for target.
reduced_path = pines_path / ('Objects/' + short_name + '/reduced')
reduced_filenames = natsort.natsorted(
[x.name for x in reduced_path.glob('*red.fits')])
reduced_files = np.array([reduced_path / i for i in reduced_filenames])
#source_names = natsort.natsorted(list(set([i.replace('X','').replace('Y','').replace('Centroid Warning','').strip() for i in centroided_sources.keys() if i != 'Filename'])))
source_names = get_source_names(centroided_sources)
#Create output plot directories for each source.
if plots:
#Camera angles for surface plots
azim_angles = np.linspace(0, 360 * 1.5, len(reduced_files)) % 360
elev_angles = np.zeros(len(azim_angles)) + 25
for name in source_names:
#If the folders are already there, delete them.
source_path = (
pines_path /
('Objects/' + short_name + '/aper_phot/' + name + '/'))
if source_path.exists():
shutil.rmtree(source_path)
#Create folders.
os.mkdir(source_path)
#Loop over all aperture radii.
for ap in ap_radii:
print(
'Doing fixed aperture photometry for {}, aperture radius = {:1.1f} pix, inner annulus radius = {} pix, outer annulus radius = {} pix.'
.format(target, ap, an_in, an_out))
#Declare a new dataframe to hold the information for all targets for this aperture.
columns = [
'Filename', 'Time UT', 'Time JD UTC', 'Time BJD TDB', 'Airmass',
'Seeing'
]
for i in range(0, len(source_names)):
columns.append(source_names[i] + ' Flux')
columns.append(source_names[i] + ' Flux Error')
columns.append(source_names[i] + ' Background')
columns.append(source_names[i] + ' Interpolation Flag')
ap_df = pd.DataFrame(index=range(len(reduced_files)), columns=columns)
output_filename = pines_path / (
'Objects/' + short_name + '/aper_phot/' + short_name +
'_fixed_aper_phot_{:1.1f}_pix_radius.csv'.format(float(ap)))
#Loop over all images.
pbar = ProgressBar()
for j in pbar(range(len(reduced_files))):
data = fits.open(reduced_files[j])[0].data
#Read in some supporting information.
log_path = pines_path / (
'Logs/' + reduced_files[j].name.split('.')[0] + '_log.txt')
log = pines_log_reader(log_path)
log_ind = np.where(
log['Filename'] == reduced_files[j].name.split('_')[0] +
'.fits')[0][0]
header = fits.open(reduced_files[j])[0].header
date_obs = header['DATE-OBS']
#Catch a case that can cause datetime strptime to crash; Mimir headers sometimes have DATE-OBS with seconds specified as 010.xx seconds, when it should be 10.xx seconds.
if len(date_obs.split(':')[-1].split('.')[0]) == 3:
date_obs = date_obs.split(':')[0] + ':' + date_obs.split(
':')[1] + ':' + date_obs.split(':')[-1][1:]
if date_obs.split(':')[-1] == '60.00':
date_obs = date_obs.split(':')[0] + ':' + str(
int(date_obs.split(':')[1]) + 1) + ':00.00'
#Keep a try/except clause here in case other unknown DATE-OBS formats pop up.
try:
date = datetime.datetime.strptime(date_obs,
'%Y-%m-%dT%H:%M:%S.%f')
except:
print(
'Header DATE-OBS format does not match the format code in strptime! Inspect/correct the DATE-OBS value.'
)
pdb.set_trace()
#Get the closest date master_dark_stddev image for this exposure time.
#We'll use this to measure read noise and dark current.
date_str = date_obs.split('T')[0].replace('-', '')
master_dark_stddev = master_dark_stddev_chooser(
pines_path / ('Calibrations/Darks/Master Darks Stddev/'),
header)
days = date.day + hmsm_to_days(date.hour, date.minute, date.second,
date.microsecond)
jd = date_to_jd(date.year, date.month, days)
ap_df['Filename'][j] = reduced_files[j].name
ap_df['Time UT'][j] = header['DATE-OBS']
ap_df['Time JD UTC'][j] = jd
ap_df['Time BJD TDB'][j] = jd_utc_to_bjd_tdb(
jd, header['TELRA'], header['TELDEC']
) #Using the telescope ra and dec should be accurate enough for our purposes
ap_df['Airmass'][j] = header['AIRMASS']
ap_df['Seeing'][j] = log['X seeing'][log_ind]
#If the shift quality has been flagged, skip this image.
if log['Shift quality flag'].iloc[log_ind] == 1:
continue
#Get the source positions in this image.
positions = []
for i in range(len(source_names)):
positions.append((float(centroided_sources[source_names[i] +
' Image X'][j]),
float(centroided_sources[source_names[i] +
' Image Y'][j])))
#Create an aperture centered on this position with radius = ap.
try:
apertures = CircularAperture(positions, r=ap)
except:
pdb.set_trace()
#Create an annulus centered on this position.
annuli = CircularAnnulus(positions, r_in=an_in, r_out=an_out)
photometry_tbl = iraf_style_photometry(apertures, annuli,
data * gain,
master_dark_stddev * gain,
header, ap_df['Seeing'][j])
for i in range(len(photometry_tbl)):
ap_df[source_names[i] + ' Flux'][j] = photometry_tbl['flux'][i]
ap_df[source_names[i] +
' Flux Error'][j] = photometry_tbl['flux_error'][i]
ap_df[source_names[i] +
' Background'][j] = photometry_tbl['background'][i]
ap_df[source_names[i] + ' Interpolation Flag'][j] = int(
photometry_tbl['interpolation_flag'][i])
#Make surface plots.
if plots:
for i in range(len(photometry_tbl)):
x_p = photometry_tbl['X'][i]
y_p = photometry_tbl['Y'][i]
fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
xx, yy = np.meshgrid(
np.arange(int(x_p) - 10,
int(x_p) + 10 + 1),
np.arange(int(y_p) - 10,
int(y_p) + 10 + 1))
theta = np.linspace(0, 2 * np.pi, 201)
y_circ = ap * np.cos(theta) + y_p
x_circ = ap * np.sin(theta) + x_p
vmin = np.nanmedian(data[yy, xx])
vmax = vmin + 2.5 * np.nanstd(data[yy, xx])
ax.plot_surface(xx,
yy,
data[yy, xx],
cmap=cm.viridis,
alpha=0.8,
rstride=1,
cstride=1,
edgecolor='k',
lw=0.2,
vmin=vmin,
vmax=vmax)
current_z = ax.get_zlim()
ax.set_zlim(current_z[0] - 150, current_z[1])
current_z = ax.get_zlim()
cset = ax.contourf(xx,
yy,
data[yy, xx],
zdir='z',
offset=current_z[0],
cmap=cm.viridis)
ax.plot(x_circ,
y_circ,
np.zeros(len(x_circ)) + current_z[0],
color='r',
lw=2,
zorder=100)
ax.set_xlabel('X')
ax.set_ylabel('Y')
ax.set_zlabel('Counts')
ax.set_title('SURFACE DIAGNOSTIC PLOT, ' + ', Ap. = ' +
str(ap) + '\n' + source_names[i] + ', ' +
reduced_files[j].name + ' (image ' +
str(j + 1) + ' of ' +
str(len(reduced_files)) + ')')
ax.view_init(elev=elev_angles[j], azim=azim_angles[j])
plot_output_path = (
pines_path /
('Objects/' + short_name + '/aper_phot/' +
source_names[i] + '/' + str(j).zfill(4) + '.jpg'))
plt.tight_layout()
plt.savefig(plot_output_path)
plt.close()
#Write output to file.
print('Saving ap = {:1.1f} aperture photometry output to {}.'.format(
ap, output_filename))
print('')
with open(output_filename, 'w') as f:
for j in range(len(ap_df)):
#Write in the header.
if j == 0:
f.write(
'{:>21s}, {:>22s}, {:>17s}, {:>17s}, {:>7s}, {:>7s}, '.
format('Filename', 'Time UT', 'Time JD UTC',
'Time BJD TDB', 'Airmass', 'Seeing'))
for i in range(len(source_names)):
if i != len(source_names) - 1:
f.write(
'{:>22s}, {:>28s}, {:>28s}, {:>34s}, '.format(
source_names[i] + ' Flux',
source_names[i] + ' Flux Error',
source_names[i] + ' Background',
source_names[i] + ' Interpolation Flag'))
else:
f.write(
'{:>22s}, {:>28s}, {:>28s}, {:>34s}\n'.format(
source_names[i] + ' Flux',
source_names[i] + ' Flux Error',
source_names[i] + ' Background',
source_names[i] + ' Interpolation Flag'))
#Write in Filename, Time UT, Time JD, Airmass, Seeing values.
format_string = '{:21s}, {:22s}, {:17.9f}, {:17.9f}, {:7.2f}, {:7.1f}, '
#If the seeing value for this image is 'nan' (a string), convert it to a float.
#TODO: Not sure why it's being read in as a string, fix that.
if type(ap_df['Seeing'][j]) == str:
ap_df['Seeing'][j] = float(ap_df['Seeing'][j])
#Do a try/except clause for writeout, in case it breaks in the future.
try:
f.write(
format_string.format(ap_df['Filename'][j],
ap_df['Time UT'][j],
ap_df['Time JD UTC'][j],
ap_df['Time BJD TDB'][j],
ap_df['Airmass'][j],
ap_df['Seeing'][j]))
except:
print(
'Writeout failed! Inspect quantities you are trying to write out.'
)
pdb.set_trace()
#Write in Flux, Flux Error, and Background values for every source.
for i in range(len(source_names)):
if i != len(source_names) - 1:
format_string = '{:22.5f}, {:28.5f}, {:28.5f}, {:34d}, '
else:
format_string = '{:22.5f}, {:28.5f}, {:28.5f}, {:34d}\n'
try:
f.write(
format_string.format(
ap_df[source_names[i] + ' Flux'][j],
ap_df[source_names[i] + ' Flux Error'][j],
ap_df[source_names[i] + ' Background'][j],
ap_df[source_names[i] +
' Interpolation Flag'][j]))
except:
if i != len(source_names) - 1:
format_string = '{:22.5f}, {:28.5f}, {:28.5f}, {:34f}, '
else:
format_string = '{:22.5f}, {:28.5f}, {:28.5f}, {:34f}\n'
f.write(
format_string.format(
ap_df[source_names[i] + ' Flux'][j],
ap_df[source_names[i] + ' Flux Error'][j],
ap_df[source_names[i] + ' Background'][j],
ap_df[source_names[i] +
' Interpolation Flag'][j]))
print('')
return
def airmass_plot(target, centroided_sources):
pines_path = pines_dir_check()
short_name = short_name_creator(target)
#Get plot style parameters.
title_size, axis_title_size, axis_ticks_font_size, legend_font_size = plot_style(
)
#Get list of souce names in the centroid output.
centroided_sources.columns = centroided_sources.keys().str.strip()
airmasses = np.array(centroided_sources['Airmass'])
times_full = np.array(centroided_sources['Time (JD UTC)'])
night_inds = night_splitter(times_full)
num_nights = len(night_inds)
times_nights = [times_full[night_inds[i]] for i in range(num_nights)]
standard_x = standard_x_range(times_nights)
fig, ax = plt.subplots(nrows=1,
ncols=num_nights,
figsize=(17, 5),
sharey=True)
for i in range(num_nights):
if i == 0:
ax[i].set_ylabel('Airmass', fontsize=axis_title_size)
inds = night_inds[i]
ax[i].plot(times_full[inds],
airmasses[inds],
marker='.',
linestyle='',
color='m',
alpha=0.3,
label='Raw airmass')
ax[i].tick_params(labelsize=axis_ticks_font_size)
ax[i].set_xlabel('Time (JD UTC)', fontsize=axis_title_size)
ax[i].grid(alpha=0.2)
ax[i].set_xlim(
np.mean(times_full[inds]) - standard_x / 2,
np.mean(times_full[inds] + standard_x / 2))
#bin
block_inds = block_splitter(times_full[inds])
block_x = np.zeros(len(block_inds))
block_y = np.zeros(len(block_inds))
block_y_err = np.zeros(len(block_inds))
for j in range(len(block_inds)):
block_x[j] = np.mean(times_full[inds][block_inds[j]])
block_y[j] = np.mean(airmasses[inds][block_inds[j]])
block_y_err[j] = np.std(airmasses[inds][block_inds[j]]) / np.sqrt(
len(airmasses[inds][block_inds[j]]))
ax[i].errorbar(block_x,
block_y,
block_y_err,
marker='o',
linestyle='',
color='m',
ms=8,
mfc='none',
mew=2,
label='Bin airmass')
#Interpolate each night's seeing.
fit_times = np.linspace(block_x[0], block_x[-1], 1000)
interp = CubicSpline(block_x, block_y)
interp_fit = interp(fit_times)
ax[i].plot(fit_times,
interp_fit,
color='c',
lw=2,
zorder=0,
alpha=0.7,
label='CS Interp.')
ax[i].legend(bbox_to_anchor=(1.005, 0.5), fontsize=legend_font_size)
plt.suptitle(short_name + ' Airmass Measurements', fontsize=title_size)
plt.subplots_adjust(left=0.07, wspace=0.05, top=0.92, bottom=0.17)
output_filename = pines_path / ('Objects/' + short_name +
'/analysis/diagnostic_plots/' +
short_name + '_airmasses.png')
plt.savefig(output_filename, dpi=300)
return
def dv_report(target, pat_version='1.0'):
def page_header():
today = date.today()
date_str = today.strftime('%b %d, %Y')
time = datetime.now()
time_str = time.strftime('%H:%M')
canvas.setFont('Times-Italic', 8) #Font for captions
canvas.setFillColor('Grey')
header_text = short_name+' DV Report, Compiled on '+date_str+' at '+time_str+' with PINES Analysis Toolkit V. '+str(pat_version)
canvas.drawCentredString(612*0.5, 792-15, header_text)
def page_footer():
canvas.setFont('Times-Italic', 8) #Font for captions
canvas.setFillColor('Grey')
footer_text = str(page_num)
canvas.drawCentredString(612*0.95, 792*0.02, footer_text)
def title_page():
page_header()
global fig_num, page_num
canvas.setFont('Times-Roman', 20) #Font for captions
canvas.setFillColor('Black')
title_text = 'PINES DV Report'
canvas.drawCentredString(fig_x*0.5, fig_y*0.9, title_text)
object_text = target+' ('+short_name+')'
canvas.setFont('Times-Roman', 16) #Font for captions
canvas.drawCentredString(fig_x*0.5, fig_y*0.87, object_text)
#Add the source image.
source_image_path = source_path/('target_and_refs.png')
img = ImageReader(source_image_path)
y = 50
w = fig_x #All the way across the page
h = w * source_aspect_ratio
canvas.drawImage(img, x, y, x+w, h)
canvas.setFont('Times-Roman', 12) #Font for captions
caption_text = 'Figure {}: Target and references.'.format(fig_num)
canvas.drawCentredString(x+w*0.5, 30, caption_text)
fig_num += 1
page_footer()
canvas.showPage()
page_num += 1
def night_page():
global fig_num, page_num
#Get all nightly-normalized target lightcurve plots in the analysis directory.
nightly_glob = natsorted(np.array([x for x in analysis_path.glob('*nightly*target_flux.png')]))
if os.path.exists(analysis_path/('optimal_aperture.txt')):
with open(analysis_path/('optimal_aperture.txt'), 'r') as f:
lines = f.readlines()
best_ap = lines[0].split(' ')[1].split('\n')[0]
ap_type = best_ap.split('_')[1]
best_aper_phot_path = analysis_path/('aper_phot_analysis/'+best_ap)
nightly_glob = np.array(natsorted([x for x in best_aper_phot_path.glob('*nightly*.png')]))
#Sort plots in order we want them in the report: arget lc, raw flux, normalized flux
nightly_glob = np.array([nightly_glob[2], nightly_glob[1], nightly_glob[0]])
if ap_type == 'fixed':
rad = best_ap.split('_')[0]
cap_ender = 'radius = {} pixels'.format(rad)
elif ap_type == 'variable':
fact = best_ap.split('_')[0]
cap_ender = 'multiplicative seeing factor = {}'.format(fact)
page_footer
caption_texts = ['Best nightly-normalized corrected target flux, {} aperture photometry, {}.'.format(ap_type, cap_ender),
'Raw flux, {} aperture photometry, {}.'.format(ap_type, cap_ender),
'Nightly-normalized flux, {} aperture photometry, {}.'.format(ap_type, cap_ender)]
for i in range(len(nightly_glob)):
if i == 0:
page_header()
canvas.setFont('Times-Roman', 12) #Font for captions
canvas.setFillColor('Black')
image_path = str(nightly_glob[i])
aperture_radius = image_path.split('=')[1].split('_')[0]
img = ImageReader(image_path)
y = (3-(i+1))*fig_y/3 + 50
w = fig_x #All the way across the page
h = w * lc_aspect_ratio
canvas.drawImage(img, x, y, x+w, h)
caption_text = 'Figure {}: '.format(fig_num)+caption_texts[i]
canvas.drawCentredString(x+w*0.5,y-h*0.1, caption_text)
fig_num += 1
if i == len(nightly_glob) - 1:
page_footer()
canvas.showPage()
page_num += 1
def global_page():
global fig_num, page_num
if os.path.exists(analysis_path/('optimal_aperture.txt')):
with open(analysis_path/('optimal_aperture.txt'), 'r') as f:
lines = f.readlines()
best_ap = lines[1].split(' ')[1]
ap_type = best_ap.split('_')[1]
best_aper_phot_path = analysis_path/('aper_phot_analysis/'+best_ap)
global_glob = np.array(natsorted([x for x in best_aper_phot_path.glob('*global*.png')]))
#Sort plots in order we want them in the report: arget lc, raw flux, normalized flux
global_glob = np.array([global_glob[2], global_glob[1], global_glob[0]])
if ap_type == 'fixed':
rad = best_ap.split('_')[0]
cap_ender = 'radius = {} pixels'.format(rad)
elif ap_type == 'variable':
fact = best_ap.split('_')[0]
cap_ender = 'multiplicative seeing factor = {}'.format(fact)
page_footer
caption_texts = ['Best globally-normalized corrected target flux, {} aperture photometry, {}.'.format(ap_type, cap_ender),
'Raw flux, {} aperture photometry, {}.'.format(ap_type, cap_ender),
'Globally-normalized flux, {} aperture photometry, {}.'.format(ap_type, cap_ender)]
for i in range(len(global_glob)):
if i == 0:
page_header()
canvas.setFont('Times-Roman', 12) #Font for captions
canvas.setFillColor('Black')
image_path = str(global_glob[i])
aperture_radius = image_path.split('=')[1].split('_')[0]
img = ImageReader(image_path)
y = (3-(i+1))*fig_y/3 + 50
w = fig_x #All the way across the page
h = w * lc_aspect_ratio
canvas.drawImage(img, x, y, x+w, h)
caption_text = 'Figure {}: '.format(fig_num)+caption_texts[i]
canvas.drawCentredString(x+w*0.5,y-h*0.1, caption_text)
fig_num += 1
if i == len(global_glob) - 1:
page_footer()
canvas.showPage()
page_num += 1
def centroid_page():
global fig_num, page_num
page_header()
canvas.setFont('Times-Roman', 12) #Font for captions
canvas.setFillColor('Black')
cutout_position_path = diagnostic_plot_path/(short_name+'_cutout_positions.png')
img = ImageReader(cutout_position_path)
y = 2*fig_y/3 - 100
w = fig_x #All the way across the page
h = w * centroid_aspect_ratio
canvas.drawImage(img, x, y, x+w, h)
caption_text = 'Figure {}: '.format(fig_num)+'Cutout image positions for all sources.'
canvas.drawCentredString(x+w*0.5,y-h*0.05, caption_text)
fig_num += 1
image_position_path = diagnostic_plot_path/(short_name+'_image_positions.png')
img = ImageReader(image_position_path)
y = 1*fig_y/3 - 200
w = fig_x #All the way across the page
h = w * centroid_aspect_ratio
canvas.drawImage(img, x, y, x+w, h)
caption_text = 'Figure {}: '.format(fig_num)+'Absolute image positions for the target.'
canvas.drawCentredString(x+w*0.5,y-h*0.05, caption_text)
fig_num += 1
page_footer()
canvas.showPage()
page_num += 1
def diagnostics_page():
global fig_num, page_num
page_header()
canvas.setFont('Times-Roman', 12) #Font for captions
canvas.setFillColor('Black')
seeing_path = diagnostic_plot_path/(short_name+'_seeing.png')
img = ImageReader(seeing_path)
y = 2*fig_y/3 +50
w = fig_x #All the way across the page
h = w * lc_aspect_ratio
canvas.drawImage(img, x, y, x+w, h)
caption_text = 'Figure {}: '.format(fig_num)+'Seeing measurements (in arcsec).'
canvas.drawCentredString(x+w*0.5,y-h*0.1, caption_text)
fig_num += 1
background_path = diagnostic_plot_path/(short_name+'_backgrounds.png')
img = ImageReader(background_path)
y = 1*fig_y/3 + 50
w = fig_x #All the way across the page
h = w * lc_aspect_ratio
canvas.drawImage(img, x, y, x+w, h)
caption_text = 'Figure {}: '.format(fig_num)+'Background measurements (in ADU). Non-linear effects begin near 4000 ADU.'
canvas.drawCentredString(x+w*0.5,y-h*0.1, caption_text)
fig_num += 1
airmass_path = diagnostic_plot_path/(short_name+'_airmasses.png')
img = ImageReader(airmass_path)
y = 0*fig_y/3 + 50
w = fig_x #All the way across the page
h = w * lc_aspect_ratio
canvas.drawImage(img, x, y, x+w, h)
caption_text = 'Figure {}: '.format(fig_num)+'Airmass measurements.'
canvas.drawCentredString(x+w*0.5,y-h*0.1, caption_text)
fig_num += 1
page_footer()
canvas.showPage()
def corr_refs_pages():
global fig_num, page_num
if os.path.exists(analysis_path/('optimal_aperture.txt')):
with open(analysis_path/('optimal_aperture.txt'), 'r') as f:
lines = f.readlines()
best_ap = lines[0].split(' ')[1].split('\n')[0]
ap_type = best_ap.split('_')[1]
best_aper_phot_path = analysis_path/('aper_phot_analysis/'+best_ap+'/corr_ref_plots/')
corr_glob = np.array(natsorted(list([x for x in best_aper_phot_path.glob('*.png')])))
count = 0
for i in range(len(corr_glob)):
if count % 3 == 0:
page_header()
canvas.setFont('Times-Roman', 12) #Font for captions
canvas.setFillColor('Black')
image_path = str(corr_glob[i])
img = ImageReader(image_path)
y = (3-((count%3)+1))*fig_y/3 + 50
w = fig_x #All the way across the page
h = w * lc_aspect_ratio
canvas.drawImage(img, x, y, x+w, h)
if i == 0:
caption_text = 'Figure {}: '.format(fig_num)+image_path.split('/')[-1].split('_')[0]+' corrected flux.'
else:
caption_text = 'Figure {}: '.format(fig_num)+'Reference '+str(i)+' corrected flux.'
canvas.drawCentredString(x+w*0.5,y-h*0.1, caption_text)
fig_num += 1
count += 1
if count % 3 == 0:
page_footer()
canvas.showPage()
page_num += 1
#Set up pathing.
pines_path = pines_dir_check()
short_name = short_name_creator(target)
print('Generating PINES DV report for {}.'.format(short_name))
output_filename = (short_name+'_dv_report.pdf').replace(' ','')
target_path = pines_path/('Objects/'+short_name)
source_path = target_path/('sources/')
analysis_path = target_path/('analysis/')
diagnostic_plot_path = analysis_path/('diagnostic_plots/')
output_path = target_path/('output/'+output_filename)
#Set up the DV report
canvas = Canvas(str(output_path), pagesize=LETTER)
global fig_num, page_num, fig_x, fig_y, source_aspect_ratio, lc_aspect_ratio, centroid_aspect_ratio, x
fig_num = 1 #Initialize
page_num = 1
fig_x = 612
fig_y = 792 #Dimensions (in pixels?) for 8.5x11 page
source_aspect_ratio = 9/10
lc_aspect_ratio = 5/17
centroid_aspect_ratio = 27/51
x = 0 #Start all figures at the left margin
#------------------------------------------------------------------------------------------------------------------------
#PAGE 1: Title and Source Detection Image.
#Make the title page.
title_page()
#------------------------------------------------------------------------------------------------------------------------
#PAGE 2: Best nightly-normalized lightcurve, raw flux, and normalized flux.
night_page()
#------------------------------------------------------------------------------------------------------------------------
#PAGE 3: Best globally-normalized lightcurve, raw flux, and normalized flux.
global_page()
#------------------------------------------------------------------------------------------------------------------------
#PAGE 4: Centroid diagnostic plots.
centroid_page()
#------------------------------------------------------------------------------------------------------------------------
#PAGE 5: Diagnostic plots.
diagnostics_page()
#------------------------------------------------------------------------------------------------------------------------
#PAGES 6-N: Plots of corrected target flux.
corr_refs_pages()
canvas.save()
return
def observed_sample_plots(upload=True):
def plot_mwd(RA,
Dec,
observed_flag,
org=0,
title='Mollweide projection',
projection='mollweide',
observed_plot=0):
'''
Plots targets on the sky in a 'Mollweide' projection.
RA, Dec are arrays of the same length.
RA takes values in [0,360), Dec in [-90,90],
which represent angles in degrees.
org is the origin of the plot, 0 or a multiple of 30 degrees in [0,360).
title is the title of the figure.
projection is the kind of projection: 'mollweide', 'aitoff', 'hammer', 'lambert'
'''
x = np.remainder(RA + 360 - org, 360) # shift RA values
ind = x > 180
x[ind] -= 360 # scale conversion to [-180, 180]
x = -x # reverse the scale: East to the left
x_tick_labels = np.array(
[150, 120, 90, 60, 30, 0, 330, 300, 270, 240,
210]) #Label in degrees
#x_tick_labels = np.array([150,140,130,120,110,100,90,80,70,60,50,40,30,20,10,0,350,340,330,320,310,300,290,280,270,260,250,240,230,220,210]) #FinerLabel in degrees
x_tick_labels = np.remainder(x_tick_labels + 360 + org, 360)
# x_tick_labels = np.array([150, 120, 90, 60, 30, 0, 330, 300, 270, 240, 210])/15 #Label in hours
# x_tick_labels = np.remainder(x_tick_labels+24+org/15,24)
x_tick_labels = [int(i) for i in x_tick_labels]
fig = plt.figure(figsize=(15 * .8, 7 * .8))
ax = fig.add_subplot(111, projection=projection)
#ax.scatter(np.radians(x),np.radians(Dec),color=color,alpha=0.4,zorder=1, label='Targets') # convert degrees to radians
for i in range(len(x)):
if np.array(observed_flag)[i] == 0:
color = 'k'
else:
color = 'k'
if observed_plot == 1:
color = 'g' #Turn on observed targest plotting.
ax.scatter(np.radians(x[i]),
np.radians(Dec[i]),
color=color,
alpha=0.4,
zorder=1,
s=25)
ax.set_yticklabels([
str(int(i)) + '$^\circ$'
for i in np.round(ax.get_yticks() * 180 / np.pi)
],
fontsize=15)
ax.title.set_fontsize(20)
ax.set_xlabel('RA')
ax.xaxis.label.set_fontsize(20)
ax.set_ylabel("Dec")
ax.yaxis.label.set_fontsize(20)
ax.set_xticklabels([], fontsize=16) # we add the scale on the x axis
ax.grid(True, alpha=0.3)
month_texts = [
'Sep', 'Aug', 'Jul', 'Jun', 'May', 'Apr', 'Mar', 'Feb', 'Jan',
'Dec', 'Nov', 'Oct'
]
for i in range(len(month_texts)):
ax.text(-180 * np.pi / 180 + 15 * np.pi / 180 +
30 * np.pi / 180 * i,
-35 * np.pi / 180,
month_texts[i],
ha='center',
va='center',
fontsize=14)
for i in range(len(x_tick_labels)):
ax.text(-150 * np.pi / 180 + 30 * np.pi / 180 * i,
-22.5 * np.pi / 180,
str(x_tick_labels[i]) + '$^\circ$',
ha='center',
va='center',
fontsize=15)
#Plot monsoon season.
monsoon_x_vertices = np.array([-150, -150, -90, -90, -150
]) * np.pi / 180
monsoon_y_vertices = np.array([-90, 90, 90, -90, -90]) * np.pi / 180
monsoon_polygon = Polygon(np.array(
[[monsoon_x_vertices[i], monsoon_y_vertices[i]]
for i in range(len(monsoon_x_vertices))]),
color='r',
alpha=0.15,
label='Flagstaff monsoon season')
ax.add_patch(monsoon_polygon)
plt.show()
return ax
'''Plots the current sample as given in 'PINES sample.xlsx' on Google drive and uploads to the PINES website.'''
pines_path = pines_dir_check()
sample_path = pines_path / ('Misc/PINES Sample.xlsx')
print('Make sure an up-to-date copy of PINES Sample.xlsx exists in {}.'.
format(pines_path / 'Misc/'))
print('Download from the PINES Google Drive.\n')
df = pd.read_excel(sample_path)
df = df.dropna(how='all') #Remove rows that are all NaNs.
good_locs = np.where(df['Good'] == 1)[0] #Get only "good" targets
ra = np.array(df['RA (deg)'][good_locs])
dec = np.array(df['Dec (deg)'][good_locs])
group_ids = df['Group ID'][good_locs]
observed_flag = df['Observed?'][good_locs]
observed_groups = np.unique(
np.array(group_ids)[np.where(
observed_flag != 0)[0]]) #Get the groups that have been observed.
number_observed = len(np.array(group_ids)[np.where(observed_flag != 0)[0]])
#Plot 1: Sky map of good targets based on group.
print('Updating sky plot...')
ax = plot_mwd(ra,
dec,
observed_flag,
org=180,
projection='mollweide',
observed_plot=1)
handles, labels = plt.gca().get_legend_handles_labels()
by_label = dict(zip(labels, handles))
ax.legend(by_label.values(),
by_label.keys(),
loc=1,
bbox_to_anchor=(1.1, 1.1),
fontsize=16)
ax.grid(alpha=0.2)
group_id_inds = np.arange(0, max(group_ids) + 1)
#Now loop over group_id inds, and draw boundaries around each group.
for i in group_id_inds:
targs_in_group = np.where(group_ids == i)[0]
try:
cluster_coords = np.array([[ra[i], dec[i]]
for i in targs_in_group])
except:
pdb.set_trace()
hull = ConvexHull(cluster_coords)
for s in range(len(hull.simplices)):
simplex = hull.simplices[s]
x = np.remainder(cluster_coords[simplex, 0] + 360 - 180,
360) # shift RA values
ind = x > 180
x[ind] -= 360 # scale conversion to [-180, 180]
x = -x # reverse the scale: East to the left
if i in observed_groups:
color = 'g'
ax.plot(x * np.pi / 180,
cluster_coords[simplex, 1] * np.pi / 180,
color=color,
lw=2,
zorder=0,
alpha=0.6,
label='Observed')
else:
color = 'k'
ax.plot(x * np.pi / 180,
cluster_coords[simplex, 1] * np.pi / 180,
color=color,
lw=2,
zorder=0,
alpha=0.6,
label='Not yet observed')
ax.grid(alpha=0.4)
handles, labels = plt.gca().get_legend_handles_labels()
by_label = dict(zip(labels, handles))
ax.legend(by_label.values(),
by_label.keys(),
loc=1,
bbox_to_anchor=(0.65, 0.225))
ax.set_title('PINES sample \n ' + str(int(max(group_ids) + 1)) +
' groups, ' + str(len(good_locs)) + ' targets',
fontsize=20)
plt.tight_layout()
sky_map_output_path = pines_path / ('Misc/updated_sky_plot.png')
plt.savefig(sky_map_output_path, dpi=300)
plt.close()
ntargs = len(df)
#Now do magnitude/SpT histograms
print('Updating target histograms...')
mags = np.zeros(ntargs)
observed_SpTs = []
observed_mags = []
SpT = []
for i in range(ntargs):
try:
#mags[i] = float(df['2MASS H'][i][0:6])
mags[i] = float(df['2MASS J'][i][0:6])
SpT.append(df['SpT'][i])
if df['Observed?'][i] != 0:
observed_SpTs.append(df['SpT'][i])
observed_mags.append(mags[i])
except: #Some values don't follow the normal +/- convention (they were upper limits in the Gagne sheet), so have to read them in differently.
#mags[i] = float(df['2MASS H'][i])
mags[i] = float(df['2MASS J'][i])
SpT.append(df['SpT'][i])
if df['Observed?'][i] != 0:
observed_SpTs.append(df['SpT'][i])
observed_mags.append(mags[i])
mags = mags[good_locs]
SpT = np.array(SpT)
observed_SpTs = np.array(observed_SpTs)
observed_mags = np.array(observed_mags)
SpT = SpT[good_locs]
SpT_number = np.zeros(ntargs)
observed_SpT_numbers = []
for i in range(ntargs):
if df['SpT'][i][0] == 'L':
SpT_number[i] = float(df['SpT'][i][1:])
if df['Observed?'][i] != 0:
observed_SpT_numbers.append(SpT_number[i])
else:
SpT_number[i] = 10 + float(df['SpT'][i][1:])
if df['Observed?'][i] != 0:
observed_SpT_numbers.append(SpT_number[i])
SpT_number = SpT_number[good_locs]
SpT_number = np.array(SpT_number)
observed_SpT_numbers = np.array(observed_SpT_numbers)
median_mag = np.median(mags)
scale_factor = 0.5
fig, ax = plt.subplots(nrows=2,
ncols=1,
figsize=(18 * scale_factor, 15 * scale_factor))
bins = np.array([
11.25, 11.75, 12.25, 12.75, 13.25, 13.75, 14.25, 14.75, 15.25, 15.75,
16.25, 16.75
]) - 0.25
ax[0].hist(mags,
bins=bins,
histtype='step',
lw=3,
ls='--',
label='Full sample')
ax[0].hist(observed_mags,
bins=bins,
histtype='bar',
label='Observed sample',
color='tab:blue')
ax[0].axvline(median_mag,
color='r',
label='Median $m_J$ = {:2.1f}'.format(median_mag))
ticks = [11, 11.5, 12, 12.5, 13, 13.5, 14, 14.5, 15, 15.5, 16, 16.5]
ax[0].plot()
ax[0].set_xticks(ticks)
ax[0].set_xticklabels([str(i) for i in ticks])
ax[0].set_xlabel('$m_J$', fontsize=20)
ax[0].set_ylabel('Number of targets', fontsize=20)
ax[0].tick_params(axis='both', which='major', labelsize=16)
ax[0].legend(fontsize=16, loc='upper left')
#ax[0].grid(alpha=0.2)
ax[1].hist(SpT_number,
bins=np.arange(-0.5,
max(SpT_number) + 0.5, 1),
histtype='step',
lw=3,
color='orange',
ls='--',
label='Full sample')
ax[1].hist(observed_SpT_numbers,
bins=np.arange(-0.5,
max(SpT_number) + 0.5, 1),
histtype='bar',
lw=3,
color='orange',
label='Observed sample')
ticks = np.arange(0, max(SpT_number), 1)
ax[1].set_xticks(ticks)
ax[1].set_xticklabels([
'L0', 'L1', 'L2', 'L3', 'L4', 'L5', 'L6', 'L7', 'L8', 'L9', 'T0', 'T1',
'T2', 'T3', 'T4', 'T5', 'T6', 'T7'
])
ax[1].set_xlabel('Spectral Type', fontsize=20)
ax[1].set_ylabel('Number of targets', fontsize=20)
ax[1].tick_params(axis='both', which='major', labelsize=16)
ax[1].legend(fontsize=16, loc='upper right')
#ax[1].grid(alpha=0.2)
plt.tight_layout()
histogram_output_path = pines_path / 'Misc/target_histograms.png'
plt.savefig(histogram_output_path, dpi=300)
plt.close()
#Edit the observing.html page to update the number of observed targets.
print('Updating observing.html...')
if not (pines_path / 'Misc/observing.html').exists():
print('Grabbing copy of observing.html from the PINES server.')
sftp = pines_login()
sftp.chdir('/web')
remote_path = '/web/observing.html'
local_path = pines_path / ('Misc/observing.html')
sftp.get(remote_path, local_path)
sftp.close()
with open(str(pines_path / ('Misc/observing.html')), 'r') as f:
lines = f.readlines()
edit_line_ind = np.where(
['To date, PINES has observed' in i for i in lines])[0][0]
edit_line = lines[edit_line_ind]
edit_line = edit_line.replace(
edit_line.split('<u>')[1].split('</u>')[0], str(number_observed))
lines[edit_line_ind] = edit_line
with open(str(pines_path / ('Misc/observing.html')), 'w') as f:
f.writelines(lines)
if upload:
sftp = pines_login()
print('Uploading plots and observing.html to the PINES server.')
sftp.chdir('/web/images')
sftp.put(sky_map_output_path, '/web/images/updated_sky_plot.png')
sftp.put(histogram_output_path, '/web/images/target_histograms.png')
sftp.chdir('/web')
sftp.put(pines_path / ('Misc/observing.html'), '/web/observing.html')
print('PINES website updated!')
from pines_analysis_toolkit.pwv.fyodor import download_nc, rename_nc
from pines_analysis_toolkit.utils.pines_dir_check import pines_dir_check
import pdb
ut_dates = ['20201203', '20201204', '20201205', '20201206']
url_codes = ['8042323663', '8042323665', '8042323666', '8042323878']
pines_path = pines_dir_check()
download_directory = pines_path/('Calibrations/PWV/')
if len(ut_dates) != len(url_codes):
raise RuntimeError('There must be one url code for every UT date you want to download.')
for i in range(len(ut_dates)):
date = ut_dates[i]
url = 'https://download.avl.class.noaa.gov/download/'+url_codes[i]+'/001'
download_nc(url, str(download_directory), date, n_threads=1)
rename_nc(str(download_directory/date))
def basic_psf_phot(target, centroided_sources, plots=False):
def hmsm_to_days(hour=0,min=0,sec=0,micro=0):
"""
Convert hours, minutes, seconds, and microseconds to fractional days.
"""
days = sec + (micro / 1.e6)
days = min + (days / 60.)
days = hour + (days / 60.)
return days / 24.
def date_to_jd(year,month,day):
"""
Convert a date to Julian Day.
Algorithm from 'Practical Astronomy with your Calculator or Spreadsheet',
4th ed., Duffet-Smith and Zwart, 2011.
"""
if month == 1 or month == 2:
yearp = year - 1
monthp = month + 12
else:
yearp = year
monthp = month
# this checks where we are in relation to October 15, 1582, the beginning
# of the Gregorian calendar.
if ((year < 1582) or
(year == 1582 and month < 10) or
(year == 1582 and month == 10 and day < 15)):
# before start of Gregorian calendar
B = 0
else:
# after start of Gregorian calendar
A = math.trunc(yearp / 100.)
B = 2 - A + math.trunc(A / 4.)
if yearp < 0:
C = math.trunc((365.25 * yearp) - 0.75)
else:
C = math.trunc(365.25 * yearp)
D = math.trunc(30.6001 * (monthp + 1))
jd = B + C + D + day + 1720994.5
return jd
def gaussian(p, x, y):
height, center_x, center_y, width_x, width_y, rotation = p
rotation = np.deg2rad(rotation)
x_0 = center_x * np.cos(rotation) - center_y * np.sin(rotation)
y_0 = center_x * np.sin(rotation) + center_y * np.cos(rotation)
def rotgauss(x,y):
xp = x * np.cos(rotation) - y * np.sin(rotation)
yp = x * np.sin(rotation) + y * np.cos(rotation)
g = height*np.exp(
-(((x_0-xp)/width_x)**2+
((y_0-yp)/width_y)**2)/2.)
return g
g = rotgauss(x,y)
return g
def moments(data):
total = np.nansum(data)
X, Y = np.indices(data.shape)
center_x = int(np.shape(data)[1]/2)
center_y = int(np.shape(data)[0]/2)
row = data[int(center_x), :]
col = data[:, int(center_y)]
width_x = np.nansum(np.sqrt(abs((np.arange(col.size)-center_y)**2*col))
/np.nansum(col))
width_y = np.nansum(np.sqrt(abs((np.arange(row.size)-center_x)**2*row))
/np.nansum(row))
height = np.nanmax(data)
rotation = 0.0
return height, center_x, center_y, width_x, width_y, rotation
def errorfunction(p, x, y, data):
return gaussian(p, x, y) - data
def fitgaussian(data):
params = moments(data)
X, Y = np.indices(data.shape)
mask = ~np.isnan(data)
x = X[mask]
y = Y[mask]
data = data[mask]
p, success = optimize.leastsq(errorfunction, params, args=(x, y, data))
return p
pines_path = pines_dir_check()
short_name = short_name_creator(target)
reduced_path = pines_path/('Objects/'+short_name+'/reduced/')
reduced_files = np.array(natsort.natsorted([x for x in reduced_path.glob('*.fits')]))
centroided_sources.columns = centroided_sources.columns.str.strip()
source_names = natsort.natsorted(list(set([i[0:-2].replace('X','').replace('Y','').rstrip().lstrip() for i in centroided_sources.keys()])))
#Declare a new dataframe to hold the information for all targets for this .
columns = ['Filename', 'Time UT', 'Time JD', 'Airmass', 'Seeing']
for i in range(0, len(source_names)):
columns.append(source_names[i]+' Flux')
columns.append(source_names[i]+' Flux Error')
psf_df = pd.DataFrame(index=range(len(reduced_files)), columns=columns)
output_filename = pines_path/('Objects/'+short_name+'/psf_phot/'+short_name+'_psf_phot.csv')
for i in range(len(reduced_files)):
#Read in image data/header.
file = reduced_files[i]
data = fits.open(file)[0].data
header = fits.open(file)[0].header
print('{}, image {} of {}.'.format(file.name, i+1, len(reduced_files)))
#Read in some supporting information.
log_path = pines_path/('Logs/'+file.name.split('.')[0]+'_log.txt')
log = pines_log_reader(log_path)
date_obs = header['DATE-OBS']
#Catch a case that can cause datetime strptime to crash; Mimir headers sometimes have DATE-OBS with seconds specified as 010.xx seconds, when it should be 10.xx seconds.
if len(date_obs.split(':')[-1].split('.')[0]) == 3:
date_obs = date_obs.split(':')[0] + ':' + date_obs.split(':')[1] + ':' + date_obs.split(':')[-1][1:]
#Keep a try/except clause here in case other unknown DATE-OBS formats pop up.
try:
date = datetime.datetime.strptime(date_obs, '%Y-%m-%dT%H:%M:%S.%f')
except:
print('Header DATE-OBS format does not match the format code in strptime! Inspect/correct the DATE-OBS value.')
pdb.set_trace()
days = date.day + hmsm_to_days(date.hour,date.minute,date.second,date.microsecond)
jd = date_to_jd(date.year,date.month,days)
psf_df['Filename'][i] = file.name
psf_df['Time UT'][i] = header['DATE-OBS']
psf_df['Time JD'][i] = jd
psf_df['Airmass'][i] = header['AIRMASS']
psf_df['Seeing'][i] = log['X seeing'][np.where(log['Filename'] == file.name.split('_')[0]+'.fits')[0][0]]
#Read in source centroids for this image
x = np.zeros(len(source_names))
y = np.zeros(len(source_names))
seeing = psf_df['Seeing'][i]
for j in range(len(source_names)):
source = source_names[j]
x[j] = centroided_sources[source+' X'][i]
y[j] = centroided_sources[source+' Y'][i]
#The extract_stars() function requires the input data as an NDData object.
nddata = NDData(data=data)
#Create table of good star positions
stars_tbl = Table()
stars_tbl['x'] = x
stars_tbl['y'] = y
size = 25
x, y = np.meshgrid(np.arange(0,size), np.arange(0,size))
#Extract star cutouts.
stars = extract_stars(nddata, stars_tbl, size=size)
fitter = fitting.LevMarLSQFitter()
fig, ax = plt.subplots(nrows=len(stars), ncols=3, sharex=True, sharey=True, figsize=(12,40))
#Fit a 2D Gaussian to each star.
for j in range(len(stars)):
star = stars[j]
source = source_names[j]
mmm_bkg = MMMBackground()
cutout = star.data - mmm_bkg(star.data)
#Get the star's centroid position in the cutout.
dtype = [('x_0', 'f8'), ('y_0', 'f8')]
pos = Table(data=np.zeros(1, dtype=dtype))
source_x = stars_tbl['x'][j]
source_y = stars_tbl['y'][j]
pos['x_0'] = source_x - int(source_x - size/2 + 1)
pos['y_0'] = source_y - int(source_y - size/2 + 1)
parameters = fitgaussian(cutout)
g2d_fit = gaussian(parameters, x, y)
avg, med, std = sigma_clipped_stats(cutout)
im = ax[j,0].imshow(cutout, origin='lower', vmin=med-std, vmax=med+8*std)
divider = make_axes_locatable(ax[j,0])
cax = divider.append_axes('right', size='5%', pad=0.05)
fig.colorbar(im, cax=cax, orientation='vertical')
ax[j,0].plot(pos['x_0'], pos['y_0'], 'rx')
ax[j,0].set_ylabel(source)
ax[j,0].text(pos['x_0'], pos['y_0']+1, '('+str(np.round(source_x,1))+', '+str(np.round(source_y,1))+')', color='r', ha='center')
ax[j,0].axis('off')
axins = ax[j,0].inset_axes([0.75, 0.75, 0.25, 0.25])
axins.set_yticklabels([])
axins.set_yticks([])
axins.set_xticklabels([])
axins.set_xticks([])
axins.imshow(data, origin='lower', vmin=med-std, vmax=med+8*std)
axins.plot(source_x, source_y, 'rx')
im = ax[j,1].imshow(g2d_fit, origin='lower', vmin=med-std, vmax=med+8*std)
divider = make_axes_locatable(ax[j,1])
cax = divider.append_axes('right', size='5%', pad=0.05)
fig.colorbar(im, cax=cax, orientation='vertical')
ax[j,1].axis('off')
avg, med, std = sigma_clipped_stats(cutout - g2d_fit)
im = ax[j,2].imshow(cutout - g2d_fit, origin='lower', vmin=med-std, vmax=med+8*std)
divider = make_axes_locatable(ax[j,2])
cax = divider.append_axes('right', size='5%', pad=0.05)
fig.colorbar(im, cax=cax, orientation='vertical')
ax[j,2].axis('off')
if j == 0:
ax[j,0].set_title('Data')
ax[j,1].set_title('2D Gaussian Model')
ax[j,2].set_title('Data - Model')
plt.tight_layout()
output_filename = pines_path/('Objects/'+short_name+'/basic_psf_phot/'+reduced_files[i].name.split('_')[0]+'_'+'source_modeling.pdf')
plt.savefig(output_filename)
plt.close()
return
