def pipeline_run_sim(path):
startTime = time.time()
if os.path.exists(path) == True:
location = path + '/data'
mask.MASK(path)
ref_image(location)
align_astroalign.align2(location)
print("-> Combining images using swarp method...")
psf.PSF(path)
combine_swarp.swarp(location)
psf.PSF(path)
print("\n-> Subtracting images using AIS method...")
subtract_ais.isis_sub(path)
optimize.perform_optimization(path)
print("-> Running SExtractor on residuals...")
extract.EXTRACT(path, method='indiv')
MR.MR_swarp(path)
extract.EXTRACT(path, method='MR')
endTime = time.time()
print("-> Finished!\n-> Total runtime: %.2f seconds\n" %
(endTime - startTime))
else:
print(
"\n-> Error: Unknown path entered\n-> Please enter the path to an existing exposure time directory\n-> Exiting...\n"
)
sys.exit()
def pipeline_run_sim(path, sim_x=1, sim_y=1, sim_width=30, sim=True):
import align_astroalign
from ref_image import ref_image
import combine_swarp
import extract
import subtract
import mask
import sys
import psf
import MR
import os
import time
startTime = time.time()
if os.path.exists(path) == True:
location = path + '/data'
ref_image(location)
align_astroalign.align2(location)
mask.MASK(path)
print("-> Combining images using swarp method...")
psf.PSF(path)
combine_swarp.swarp(location)
psf.PSF(path)
print("\n-> Subtracting images...")
subtract.SUBTRACT(path)
# print("\n-> Subtracting images using AIS method...")
# subtract_ais.isis_sub(path)
# optimize.perform_optimization(path, s_x=sim_x, s_y=sim_y, s_width=sim_width, simulate=sim, qFloor=0.80, qValue=0.90, qInitial=0.95, use_config_file=False)
print("-> Running SExtractor on residuals...")
extract.EXTRACT(path, method='indiv')
MR.MR(path)
extract.EXTRACT(path, method='MR')
endTime = time.time()
print("-> Finished!\n-> Total runtime: %.2f seconds\n" %
(endTime - startTime))
else:
print(
"\n-> Error: Unknown path entered\n-> Please enter the path to an existing exposure time directory\n-> Exiting...\n"
)
sys.exit()
def ALIGN(path, align_method='standard'):
'''Registers all science images to their reference image. If no reference image exists, one is chosen (see documentation for details).
:param str path: Path of data file tree (contains the **configs**, **data**, **psf**, **residuals**, **sources**, **templates** directories). Use a comma-separated list for mapping to multiple datasets.
:param str align_method: Method of alignment. Can be either *standard* or *fakes*. Default is *standard*. The *fakes* method should be used only for simulations, as it bypasses registration and only performs photometric alignment.
:returns: Aligns all science images are aligned to the reference image to subpixel precision. A succesful alignment changes an image's suffix from *_U_* to *_A_*.
'''
paths = (path.replace(' ', '')).split(',')
del path
for path in paths:
if os.path.exists(path):
location = path + '/data'
mask.MASK(path)
sat = check_saturation.check_saturate(location)
if sat == 0:
ref_image(location)
align_astroalign.align2(location, method=align_method)
else:
check = input(
"-> Saturated images found, continue image alignment? (y/n): "
)
if check == 'y':
print("-> Moving saturated images to OASIS archives...")
check_saturation.move_arch(sat)
ref_image(location)
align_astroalign.align2(location, method=align_method)
elif check == 'n':
pass
else:
print("-> Unknown input: must be y or n")
sys.exit()
else:
print(
"\n-> Error: Unknown path entered\n-> Please enter the path to an existing exposure time directory\n-> Exiting...\n"
)
sys.exit()
def PIPELINE(path):
'''The **OASIS Pipeline**. Runs all **OASIS** functions in succession.
:param str path: Path of data file tree (contains the **configs**, **data**, **psf**, **residuals**, **sources**, **templates** directories). Use a comma-separated list for mapping to multiple datasets.
:returns: All-in-one difference imaging pipeline. Raw science images are placed in the **data** directory, and residual images and source catalogs are outputted into the **residuals** and **sources** directories, respectively.
'''
paths = (path.replace(' ', '')).split(',')
del path
for path in paths:
startTime = time.time()
if os.path.exists(path) == True:
initialize.create(path)
location = path + '/data'
mask.MASK(path)
sat = check_saturation.check_saturate(location)
if sat == 0:
ref_image(location)
align_astroalign.align2(location)
else:
check = input(
"-> Saturated images found\n-> Move saturated images to OASIS archive? (y/n): "
)
if check == 'y':
check_saturation.move_arch(sat)
ref_image(location)
align_astroalign.align2(location)
elif check == 'n':
ref_image(location)
align_astroalign.align2(location)
else:
print("-> Error: Unknown input")
sys.exit()
print("-> Combining images using swarp method...")
psf.PSF(path)
combine_swarp.swarp(location)
psf.PSF(path)
print("\n-> Subtracting images using AIS method...")
subtract_ais.isis_sub(path)
optimize.perform_optimization(path)
print("-> Running SExtractor on residuals...")
extract.EXTRACT(path, method='indiv')
MR.MR(path)
extract.EXTRACT(path, method='MR')
endTime = time.time()
print("-> Finished!\n-> Total runtime: %.2f seconds\n" %
(endTime - startTime))
else:
print(
"\n-> Error: Unknown path entered\n-> Please enter the path to an existing exposure time directory\n-> Exiting...\n"
)
sys.exit()
def PIPELINE():
get_check = input("-> Get data or analyze existing data? (get/analyze): ")
if get_check == 'get':
get.GET()
elif get_check == 'analyze':
location = input("-> Enter path to data directory: ")
sat = check_saturation.check_saturate(location)
if sat == 0:
ref_image(location)
align_astroalign.align2(location)
else:
check = input(
"-> Saturated images found, continue image alignment? (y/n): ")
if check == 'y':
move = input(
"-> Move saturated images to SDI archives before continuing? (y/n): "
)
if move == 'y':
check_saturation.move_arch(sat)
ref_image(location)
align_astroalign.align2(location)
else:
ref_image(location)
align_astroalign.align2(location)
method = input(
"-> Choose combination method-- numpy (default) or swarp: ")
if method == "swarp":
combine_swarp.swarp(location)
elif method == "numpy" or method == "":
combine_numpy.combine_median(location)
else:
print("-> Error: unknown method entered")
path = location[:-5]
sub_method = input(
"\n-> Choose subtraction method-- ais (default) or hotpants: ")
if sub_method == '' or sub_method == 'ais':
subtract_ais.isis_sub(path)
elif sub_method == 'hotpants':
subtract_hotpants.hotpants(path)
else:
print("\n-> Error: Unknown method")
ask = input("-> Run sextractor on residual images? (y/n): ")
if ask == 'y':
extract.SEXTRACTOR(path)
elif ask != 'y' and ask != 'n':
print("-> Error: unknown input")
def ALIGN():
location = input("-> Enter path to data directory: ")
sat = check_saturation.check_saturate(location)
if sat == 0:
ref_image(location)
align_astroalign.align2(location)
else:
check = input("-> Saturated images found, continue image alignment? (y/n): ")
if check == 'y':
move = input("-> Move saturated images to SDI archives before continuing? (y/n): ")
if move == 'y':
check_saturation.move_arch(sat)
ref_image(location)
align_astroalign.align2(location)
elif move == 'n':
ref_image(location)
align_astroalign.align2(location)
else:
print("-> Unknown input: must be y or n")
elif check =='n':
pass
else:
print("-> Unknown input: must be y or n")
if ask == 'y':
extract.SEXTRACTOR(path)
elif ask != 'y' and ask != 'n':
print("-> Error: unknown input")
if __name__ == '__main__':
get_check = input("-> Get data or analyze existing data? (get/analyze): ")
if get_check == 'get':
get.GET()
elif get_check == 'analyze':
location = input("-> Enter path to data directory: ")
sat = check_saturation.check_saturate(location)
if sat == 0:
ref_image(location)
align_astroalign.align2(location)
else:
check = input(
"-> Saturated images found, continue image alignment? (y/n): ")
if check == 'y':
move = input(
"-> Move saturated images to SDI archives before continuing? (y/n): "
)
if move == 'y':
check_saturation.move_arch(sat)
ref_image(location)
align_astroalign.align2(location)
else:
ref_image(location)
align_astroalign.align2(location)
method = input(
def TEST():
'''Tests the installation of **OasisPy** by downloading a set of images from an online public archive, adding fake sources to one of the images, and running the dataset through the **OASIS Pipeline**.
If the fake sources are recovered, the test is a success. The images used are 118 second exposures of exoplanet HAT-P-37b taken with telescopes at the Las Cumbres Observatory.
Results of the test are compared to control results located in **OasisPy**'s source code directory.
:returns: Prints either 'TEST SUCCESSFUL!' or 'Test failure: Results do not match controls'.
'''
frameNum = 30
q_initial = 1
q_value = 0.90
q_min = 0.80
startTime = time.time()
#look for existing TEST folder and delete it
og_test = loc + '/OASIS/targets/TEST'
if os.path.exists(og_test) == True:
shutil.rmtree(og_test)
#get data from LCO public archive and put in target directory under 'TEST' folder
print("\n-> Getting data from LCO...")
object_name = 'HAT-P-37'
response = requests.get('https://archive-api.lco.global/frames/?' +
'limit=%d&' % (frameNum) + 'RLEVEL=91&' +
'PROPID=' + 'LCOEPO2014B-007' + '&' + 'OBJECT=' +
'%s&' % (object_name) + 'FILTER=' + 'w&' +
'start=' + '2019-05-29' + '&'
'end=' + '2019-05-31' + '&').json()
frames = response['results']
# print(len(frames))
#take only the first 25 frames
frames = frames[:frameNum]
#download data
temp_loc = loc + '/OASIS/temp/'
os.mkdir(temp_loc + 'test_data')
for frame in frames:
with open(temp_loc + 'test_data/' + frame['filename'], 'wb') as f:
f.write(requests.get(frame['url']).content)
#funpack and move to 'TEST' folder
obtain.process()
obtain.movetar()
old_data_location = obtain.rename()
data_location = old_data_location.replace(object_name.replace(' ', '_'),
'TEST')
os.rename("%s/OASIS/targets/%s" % (loc, object_name.replace(' ', '_')),
"%s/OASIS/targets/TEST" % (loc))
#align and combine images
test_loc = data_location[:-5]
mask.MASK(test_loc)
check_saturation.check_saturate(test_loc + '/data')
ref_image.ref_image(test_loc + '/data')
align_astroalign.align2(test_loc + '/data')
psf.PSF(test_loc)
combine_swarp.swarp(test_loc + '/data')
#get PSFs of images so fake stars with same seeing can be added
fake_im = '02:59:10.860_A_.fits'
print('\n-> Image %s chosen as fake image\n' % (fake_im))
fake_residual = fake_im.replace('_A_', '_A_residual_')
psf.PSF(test_loc)
FWHM = psf.fwhm(test_loc + '/data/%s' % (fake_im))
#add three fake stars to reference image
print("\n-> Adding fake stars to test image...")
hdu = fits.open(test_loc + '/data/%s' % (fake_im))
hdu_data = hdu[0].data
hdu_header = hdu[0].header
hdu_mask = hdu[1].data
h, w = np.shape(hdu_data)
pos_x = [1500, 200, 1200]
pos_y = [1600, 1400, 700]
array = np.array([0.65343465, 0.50675629, 0.84946314])
fluxes = (200000.0 * array) + 300.0
print("\n-> Fake locations (in pixel coords):")
print("\t X:", pos_x)
print("\t Y:", pos_y)
print("-> Fake fluxes (in ADUs):")
print("\t ", fluxes)
img = make_gaussian_im(h,
w,
fluxes=[fluxes[0], fluxes[1], fluxes[2]],
x_pos=[pos_x[0], pos_x[1], pos_x[2]],
y_pos=[pos_y[0], pos_y[1], pos_y[2]],
std=[FWHM, FWHM, FWHM])
finalData = fits.PrimaryHDU(hdu_data + img, header=hdu_header)
finalMask = fits.ImageHDU(hdu_mask)
finalList = fits.HDUList([finalData, finalMask])
finalList.writeto(test_loc + '/data/%s' % (fake_im), overwrite=True)
hdu.close()
#subtract images using ISIS
subtract_ais.isis_sub(test_loc)
perform_optimization(test_loc,
qInitial=q_initial,
qValue=q_value,
qFloor=q_min,
use_config_file=False)
MR.MR_swarp(test_loc)
#perform SExtractor on residual images
extract.EXTRACT(test_loc)
#print TEST runtime
endTime = time.time()
print("\n-> Total test runtime: %.3fs\n" % (endTime - startTime))
residual = glob.glob(test_loc + '/residuals/%s' % (fake_residual))
fake1_check = False
fake2_check = False
fake3_check = False
MR_sources, inds = filters.get_sources("%s/residuals/MR.fits" % (test_loc),
filtered=True,
MR=True)
for src in MR_sources:
if 1498 < src[1] < 1502 and 1598 < src[2] < 1602:
fake1_check = True
elif 198 < src[1] < 202 and 1398 < src[2] < 1402:
fake2_check = True
elif 1198 < src[1] < 1202 and 698 < src[2] < 702:
fake3_check = True
if fake1_check == True and fake2_check == True and fake3_check == True:
print("\n-> Test SUCCESSFUL!")
else:
print("-> Test failure: Results do not match controls")
#display final residual test image
os.system('ds9 %s -scale zscale' % (residual[0]))
def TEST():
#get data from LCO public archive and put in target directory under 'TEST' folder
print("-> Getting data from LCO...")
response = requests.get('https://archive-api.lco.global/frames/?' +
'RLEVEL=91&' +
'PROPID='+'standard&'
'OBJECT='+'L113&' +
'FILTER='+'B&'
'start='+'2018-9-14'+'&' +
'end='+'2018-9-15'+'&'
).json()
frames = response['results']
#delete bad images
del_fr = []
for fr in frames:
if fr['id'] != 9602135 and fr['id'] != 9602132:
del_fr.append(fr)
for delete in del_fr:
del frames[frames.index(delete)]
#download data
temp_loc = loc + '/sdi/temp/'
os.mkdir(temp_loc+'test_data')
for frame in frames:
with open(temp_loc + 'test_data/' + frame['filename'], 'wb') as f:
f.write(requests.get(frame['url']).content)
#funpack and move to 'TEST' folder
obtain.process()
obtain.movetar()
old_data_location = obtain.rename()
data_location = old_data_location.replace('L113', 'TEST')
os.rename(old_data_location[:29], data_location[:29])
#align and combine images
test_loc = data_location[:-5]
check_saturation.check_saturate(test_loc + '/data')
ref_image.ref_image(test_loc + '/data')
align_astroalign.align2(test_loc + '/data')
# align_skimage.skimage(test_loc + '/data')
combine_numpy.combine_median(test_loc + '/data')
# align_skimage.skimage_template(test_loc + '/data')
#add three fake stars to reference image
print("\n-> Adding fake stars to test image...")
hdu = fits.getdata(test_loc + '/data/09:14:00.260_A_.fits')
h, w = img_shape = np.shape(hdu)
pos_x = [1500,2000,1200]
pos_y = [1600,1400,2200]
array = np.array([ 0.65343465, 0.50675629, 0.84946314])
fluxes = 2000000.0 + array * 300.0
img = np.zeros(img_shape)
for x, y, f in zip(pos_x, pos_y, fluxes):
img[x, y] = f
img = gaussian_filter(img, sigma=15.0, mode='constant')
final = fits.PrimaryHDU(hdu+img)
final.writeto(test_loc + '/data/09:14:00.260_A_.fits', overwrite=True)
#subtract images using ISIS
subtract_ais.isis_sub_test(test_loc)
#get PSFs then perform SExtractor on residual images
sex.sextractor_psf(test_loc)
psf.psfex(test_loc)
sex.sextractor(test_loc)
#test the results of the test function against known values
# with open(test_loc + '/sources/sources.txt', 'r') as source:
# lines = source.readlines()
# source.close()
#
# with open(os.path.dirname(sex.__file__) + '/test_config/test_sources.txt', 'r') as test_source:
# lines2 = test_source.readlines()
# test_source.close()
res_image_loc = os.path.dirname(sex.__file__) + '/test_config/09:14:00.260_A_residual_.fits'
test_image_data = fits.getdata(res_image_loc)
residual = glob.glob(test_loc + '/residuals/*_A_residual_.fits')
residual_data = fits.getdata(residual[0])
#
# if lines == lines2:
# print("\t-> Sources matched to control")
if test_image_data.all() == residual_data.all():
print("-> Residuals matched to control\n-> TEST SUCCESSFUL!")
# if lines == lines2 and test_image_data.all() == residual_data.all():
# print("\t-> Test successful!")
if test_image_data.all() != residual_data.all():
print("-> Test failure: Results do not match controls")
#display final residual test image
os.system('ds9 %s -scale zscale' % (residual[0]))
for frame in frames:
with open(temp_loc + 'test_data/' + frame['filename'], 'wb') as f:
f.write(requests.get(frame['url']).content)
#funpack and move to 'TEST' folder
obtain.process()
obtain.movetar()
old_data_location = obtain.rename()
data_location = old_data_location.replace('L113', 'TEST')
os.rename(old_data_location[:29], data_location[:29])
#align and combine images
test_loc = data_location[:-5]
check_saturation.check_saturate(test_loc + '/data')
ref_image.ref_image(test_loc + '/data')
align_astroalign.align2(test_loc + '/data')
# align_skimage.skimage(test_loc + '/data')
combine_numpy.combine_median(test_loc + '/data')
# align_skimage.skimage_template(test_loc + '/data')
#add three fake stars to reference image
print("\n-> Adding fake stars to test image...")
hdu = fits.getdata(test_loc + '/data/09:14:00.260_A_.fits')
h, w = img_shape = np.shape(hdu)
pos_x = [1500,2000,1200]
pos_y = [1600,1400,2200]
array = np.array([ 0.65343465, 0.50675629, 0.84946314])
fluxes = 2000000.0 + array * 300.0
img = np.zeros(img_shape)
for x, y, f in zip(pos_x, pos_y, fluxes):