def int_match_to_template(location, image, template, nx=100, ny=100): print("\n-> Matching flux scale to template...") initialize.create_configs(location) linmatch_loc = location + '/configs/linmatch.txt' if os.path.exists(linmatch_loc) == False: os.system('touch %s' % (linmatch_loc)) pyraf.iraf.linmatch(image + '[0]', template + '[0]', 'grid %dx %dy' % (nx, ny), linmatch_loc, output=image + 'TEMP') temp_image = glob.glob(location + '/data/*TEMP.fits') temp_image_hdu = fits.open(temp_image[0]) temp_image_data = temp_image_hdu[0].data image_hdu = fits.open(image) image_hdr = image_hdu[0].header image_mask = image_hdu[1].data temp_image_masked = np.ma.MaskedArray(temp_image_data, mask=((image_mask - 1) * -1)) temp_image_median = np.ma.median(temp_image_masked) temp_image_hdu.close() image_hdu.close() hduData = fits.PrimaryHDU(temp_image_data, header=image_hdr) hduMask = fits.ImageHDU(image_mask) hduList = fits.HDUList([hduData, hduMask]) hduList.writeto(image, overwrite=True) median_hdu = fits.open(image, mode='update') (median_hdu[0].header).set('MEDIAN', str(temp_image_median)) median_hdu.close() os.system("rm %s" % (temp_image[0]))
def sextractor_psf_sim(location, image): psf_loc = location + "/psf" data = location + "/data" check = os.path.exists(psf_loc) length = len(data) + 1 if check == False: os.system("mkdir %s" % (psf_loc)) initialize.create_configs(location) config_loc = location + '/configs/psf.sex' with open(config_loc, 'r') as config: data = config.readlines() config.close() data[ 9] = "PARAMETERS_NAME" + " " + location + "/configs/default.psfex" + "\n" data[ 20] = "FILTER_NAME" + " " + location + "/configs/default.conv" + "\n" with open(config_loc, 'w') as config: config.writelines(data) config.close() print("\n-> Creating PSF catalog of fake image...") name = image[length:-5] with open(config_loc, 'r') as config: data = config.readlines() config.close() data[ 6] = "CATALOG_NAME" + " " + psf_loc + "/" + name + ".cat" + "\n" with open(config_loc, 'w') as config: config.writelines(data) config.close() os.system("sextractor %s[0] -c %s" % (image, config_loc))
def psfex(location): psf_loc = location + '/psf' cats = glob.glob(psf_loc + '/*.cat') initialize.create_configs(location) config_loc = location + '/configs/psfex.config' print("\n-> Calculating PSFs...\n") for cat in cats: with open(config_loc, 'r') as config: data = config.readlines() config.close() data[83] = "PSF_DIR" + " " + location + "/psf" + "\n" with open(config_loc, 'w') as config: config.writelines(data) config.close() os.system("psfex %s > %s.psf -c %s" % (cat, cat[:-4], config_loc))
def sextractor_sim(image): location = image.split('/')[:-2] location = '/'.join(location) sources = location + "/sources" check = os.path.exists(sources) check_temp = os.path.exists(sources + '/temp') if check == False: os.system("mkdir %s" % (sources)) os.system("mkdir %s/temp" % (sources)) else: if check_temp == False: os.system("mkdir %s/temp" % (sources)) initialize.create_configs(location) config_loc = location + '/configs/default.sex' with open(config_loc, 'r') as config: data = config.readlines() config.close() data[ 9] = "PARAMETERS_NAME" + " " + location + "/configs/default.param" + "\n" data[ 20] = "FILTER_NAME" + " " + location + "/configs/default.conv" + "\n" with open(config_loc, 'w') as config: config.writelines(data) config.close() print("\n-> SExtracting fake image...") name = image.split('/')[-1] with open(config_loc, 'r') as config: data = config.readlines() config.close() data[ 106] = "PSF_NAME" + " " + location + "/psf/" + name[: -5] + ".psf" + "\n" with open(config_loc, 'w') as config: config.writelines(data) config.close() os.system("sextractor %s[0]> %s/temp/%s.txt -c %s" % (image, sources, name, config_loc)) temp_hdu_data = fits.PrimaryHDU((fits.getdata(image)) * -1, header=fits.getheader(image)) temp_hdu_mask = fits.ImageHDU(fits.getdata(image, 1)) temp_hdu_list = fits.HDUList([temp_hdu_data, temp_hdu_mask]) temp_hdu_list.writeto("%s/residuals/temp.fits") os.system("sextractor %s/residuals/temp.fits[0]> %s/temp/%s.txt -c %s" % (location, sources, name, config_loc)) os.remove("%s/residuals/temp.fits" % (location)) src_join(location) filter_sources(location)
def COMBINE(path): '''Stacks science images into a high *S/N* template frame. Stacking method is the weighted median value of each pixel and is done by the AstrOmatic software ``SWarp`` (E. Bertin). Only the top third of science images with respect to seeing are included in the template. :param str path: Path of data file tree (contains the **configs**, **data**, **psf**, **residuals**, **sources**, **templates** directories). Use a comma-separated list for mapping to multiple datasets. :returns: Weighted median coaddition of all science images is outputted into the **templates** directory with the name convention of *StackMethod_NumberOfImagesInDataset.fits*. ''' paths = (path.replace(' ','')).split(',') del path for path in paths: location = path + '/data' if os.path.exists(path): initialize.create_configs(path) combine_swarp.swarp(location) else: print("\n-> Error: Unknown path entered\n-> Please enter the path to an existing exposure time directory\n-> Exiting...\n") sys.exit()
def psfex(location, images=[]): initialize.create_configs(location) config_loc = location + '/configs/psfex.config' with open(config_loc, 'r') as config: data = config.readlines() config.close() data[83] = "PSF_DIR" + " " + location + "/psf" + "\n" with open(config_loc, 'w') as config: config.writelines(data) config.close() print("\n-> Calculating PSFs...") for im in images: cat = im.replace('data','psf') cat = cat.replace('templates', 'psf') cat = cat[:-5] + '.cat' os.system("psfex %s > %s.psf -c %s" % (cat, cat[:-4], config_loc)) if images == []: print("-> PSFs already exist")
def phose_sex(location): """ create source catalogs for each science image using SExtractor, combine them into a master catalog called 'phose_sources.txt' in the data directory also creates a template catalog used for phose thresholding """ initialize.create_configs(location) try: os.mkdir("%s/data/cats" % (location)) except: pass images = glob.glob("%s/data/*.fits" % (location)) config = "%s/configs/phose_default.sex" % (location) param = "%s/configs/phose_default.param" % (location) for i in tqdm(images): output_cat = i.replace('data', 'data/cats') output_cat = output_cat.replace('fits', 'txt') with open(config, 'r') as conf: conf_lines = conf.readlines() conf_lines[9] = "PARAMETERS_NAME" + " " + param + "\n" conf_lines[32] = "WEIGHT_IMAGE" + " " + "%s[1]" % (i) + "\n" with open(config, 'w') as conf: conf.writelines(conf_lines) os.system("sextractor %s[0] > %s -c %s" % (i, output_cat, config)) sex.src_join(location, phose=True) template = glob.glob("%s/templates/*.fits" % (location)) if template == []: print( "-> Error: template not found\n-> Run combine.py first\n-> Exiting..." ) sys.exit() template = template[0] output_template_cat = "%s/templates/phose_template.cat" % (location) with open(config, 'r') as conf: conf_lines = conf.readlines() conf_lines[9] = "PARAMETERS_NAME" + " " + param + "\n" conf_lines[32] = "WEIGHT_IMAGE" + " " + "%s[1]" % (template) + "\n" with open(config, 'w') as conf: conf.writelines(conf_lines) os.system("sextractor %s[0] > %s -c %s" % (template, output_template_cat, config))
def int_match_to_ref(location, nx=100, ny=100, use_config_file=True): print("\n-> Matching flux scales to reference image...") ref_im = glob.glob(location + '/data/*_ref_A_.fits') images = glob.glob(location + '/data/*_A_.fits') initialize.create_configs(location) linmatch_loc = location + '/configs/linmatch.txt' if os.path.exists(linmatch_loc) == False: os.system('touch %s' % (linmatch_loc)) if use_config_file == True: nx = initialize.get_config_value('int_match_nx', file_loc=location + '/configs') ny = initialize.get_config_value('int_match_ny', file_loc=location + '/configs') for i in images: if i != ref_im[0]: try: fits.getval(i, 'SCALED') except KeyError: fits.setval(i, 'SCALED', value='N') if fits.getval(i, 'SCALED') == 'N': pyraf.iraf.linmatch(i + '[0]', ref_im[0] + '[0]', 'grid %dx %dy' % (nx, ny), linmatch_loc, output=i + 'TEMP') temp_image = glob.glob(location + '/data/*TEMP.fits') temp_image_hdu = fits.open(temp_image[0]) temp_image_data = temp_image_hdu[0].data image_hdu = fits.open(i, mode='update') image_hdr = image_hdu[0].header image_hdr.set('SCALED', 'Y') image_mask = image_hdu[1].data temp_image_hdu.close() image_hdu.close() image_hdr = fits.getheader(i) hduData = fits.PrimaryHDU(temp_image_data, header=image_hdr) hduMask = fits.ImageHDU(image_mask) hduList = fits.HDUList([hduData, hduMask]) hduList.writeto(i, overwrite=True) os.system("rm %s" % (temp_image[0]))
def swarp(location, template_perc=0.33, use_config_file=True): location = location[:-5] temps = glob.glob(location + '/templates/*.fits') images = glob.glob(location + '/data/*_A_.fits') imNum = len(images) numImages = 0 if use_config_file == True: template_perc = initialize.get_config_value('template_perc') if len(temps) == 1: temps_name = temps[0].split('/')[-1] numImages = int((temps_name.split('.'))[0].split('_')[-1]) if len(temps) == 0 or numImages != len(images): #delete old template if len(temps) != 0: template_name = temps[0].split('/')[-1] os.remove(temps[0]) try: os.remove("%s/psf/%s.cat" % (location, template_name[:-5])) os.remove("%s/psf/%s.psf" % (location, template_name[:-5])) except: pass #change image shapes to match each the smallest image in the set print("\n-> Slicing images to a common FOV...") shapes = [] areas = [] for i in tqdm(images): image_data = fits.getdata(i) shapes.append(image_data.shape) areas.append((image_data.shape)[0] * (image_data.shape)[1]) min_index = areas.index(min(areas)) # correct_shape = max(set(shapes), key=shapes.count) correct_shape = shapes[min_index] print("\n-> FOV size (x,y): (%d, %d)" % (correct_shape[0], correct_shape[1])) for index in tqdm(range(len(shapes))): s = shapes[index] diff = tuple(np.subtract(s, correct_shape)) im = images[index] im_hdu = fits.open(im) im_data = im_hdu[0].data im_header = im_hdu[0].header im_mask = (im_hdu[1].data).astype(int) if diff != (0, 0): if diff[0] < 0: im_data = np.concatenate( (im_data, np.zeros((diff[0] * -1, s[1]))), axis=0) im_mask = np.concatenate( (im_mask, np.ones((diff[0] * -1, s[1]))), axis=0) if diff[0] > 0: im_data = im_data[:-1 * diff[0]] im_mask = im_mask[:-1 * diff[0]] if diff[1] < 0: im_data = np.concatenate( (im_data, np.zeros((im_data.shape[0], diff[1] * -1))), axis=1) im_mask = np.concatenate( (im_mask, np.ones((im_mask.shape[0], diff[1] * -1))), axis=1) if diff[1] > 0: im_data = im_data[:, :diff[1] * -1] im_mask = im_mask[:, :diff[1] * -1] hduData = fits.PrimaryHDU(im_data, header=im_header) hduMask = fits.ImageHDU(im_mask.astype(int)) hduList = fits.HDUList([hduData, hduMask]) hduList.writeto(im, overwrite=True) im_hdu.close() #change all masks into weight maps print("\n-> Converting all image masks into weight maps...") for i in tqdm(images): weight = sex.weight_map(i) hdu = fits.open(i, mode='update') data = hdu[0].data hdr = hdu[0].header try: if hdr['WEIGHT'] == 'N': hdr.set('WEIGHT', 'Y') hduData = fits.PrimaryHDU(data, header=hdr) hduWeight = fits.ImageHDU(weight) hduList = fits.HDUList([hduData, hduWeight]) hduList.writeto(i, overwrite=True) except KeyError: hdr.set('WEIGHT', 'Y') hduData = fits.PrimaryHDU(data, header=hdr) hduWeight = fits.ImageHDU(weight) hduList = fits.HDUList([hduData, hduWeight]) hduList.writeto(i, overwrite=True) hdu.close() # choose only the top template_perc seeing images try: FWHMs = [] for im in images: FWHMs.append(psf.fwhm(im)) template_images = [] while len(template_images) < round(template_perc * len(images)): template_images.append(images[FWHMs.index(np.min(FWHMs))]) FWHMs.remove(np.min(FWHMs)) images = template_images except FileNotFoundError: print( "-> Error: PSF models do not exist, run PSF method first then try again." ) sys.exit() initialize.create_configs(location) config_loc = location + '/configs/default.swarp' if os.path.exists(config_loc): template = location + "/templates/swarp_median_" + str( imNum) + ".fits" with open(config_loc, 'r') as config: data = config.readlines() config.close() data[4] = "IMAGEOUT_NAME" + " " + template + "\n" data[ 15] = "WEIGHT_IMAGE" + " " + "@%s/templates/weights.txt" % ( location) + "\n" data[ 36] = "IMAGE_SIZE" + " " + "%s, %s" % correct_shape[:: -1] + "\n" with open(config_loc, 'w') as config: config.writelines(data) config.close() time = strftime("%Y-%m-%d %H:%M:%S", gmtime()) og_templates = glob.glob(location + "/templates/*.fits") log_loc = location + "/templates/log.txt" tlist_loc = location + "/templates/template_inputs.txt" weight_list = "%s/templates/weights.txt" % (location) log_list = open(log_loc, "a+") template_list = open(tlist_loc, "w+") for i in images: template_list.write(str(i) + "[0]" + "\n") template_list.close() with open(weight_list, 'w+') as w: for i in images: w.write("%s[1]\n" % (i)) if images == []: print("-> No aligned images to combine\n") else: try: print("-> Images being combined...\n") os.system("swarp @%s -c %s" % (tlist_loc, config_loc)) log_list.write( "template updated at %s UTC | method = median (SWarp) | images = %d\n" % (str(time), len(images))) log_list.close() if len(og_templates) > 0: for o in og_templates: os.system("mv %s %s/OASIS/archive/templates" % (o, initialize.loc)) print( "\n-> Image combination successful!\n-> Template log updated\n" ) except: print("-> Image combination failed\n") sys.exit() temp_hdu = fits.open(template) temp_data = temp_hdu[0].data temp_hdr = temp_hdu[0].header try: temp_mask = fits.getdata( os.path.dirname(initialize.__file__) + '/coadd.weight.fits') except: try: temp_mask = fits.getdata( os.path.dirname(initialize.__file__) + '/AIS_temp/coadd.weight.fits') except: print( '-> Error: can\'t find coadd.weight.fits\n-> Exiting...' ) sys.exit() mask_median = np.median(temp_mask) mask_std = np.std(temp_mask) threshold = mask_median - (mask_std) temp_mask[temp_mask < threshold] = 0 temp_mask[temp_mask >= threshold] = 1 masked_data = np.ma.masked_array(temp_data, mask=temp_mask) temp_median = np.ma.median(masked_data) temp_hduData = fits.PrimaryHDU(temp_data, header=temp_hdr) temp_hduMask = fits.ImageHDU(temp_mask) temp_hduList = fits.HDUList([temp_hduData, temp_hduMask]) temp_hduList.writeto(template, overwrite=True) temp_hdu.close() temp_hdu = fits.open(template, mode='update') (temp_hdu[0].header).set('MEDIAN', str(temp_median)) temp_hdu.close() else: print("\n-> No default.swarp file in target's config directory\n") sys.exit() else: print("-> Template already exists") try: os.remove(os.path.dirname(initialize.__file__) + '/coadd.weight.fits') except: pass
def MR_swarp(location): print("\n-> Constructing master residual...\n") residuals = glob.glob("%s/residuals/*residual_.fits" % (location)) MR_loc = "%s/residuals/MR.fits" % (location) if residuals != []: if os.path.exists(MR_loc) == False: #first change all masks into weight maps print("-> Converting all residual masks into weight maps...\n") for r in tqdm(residuals): weight = sex.weight_map(r) hdu = fits.open(r, mode='update') data = hdu[0].data hdr = hdu[0].header try: if hdr['WEIGHT'] == 'N': hdr.set('WEIGHT', 'Y') hduData = fits.PrimaryHDU(data, header=hdr) hduWeight = fits.ImageHDU(weight) hduList = fits.HDUList([hduData, hduWeight]) hduList.writeto(r, overwrite=True) except KeyError: hdr.set('WEIGHT', 'Y') hduData = fits.PrimaryHDU(data, header=hdr) hduWeight = fits.ImageHDU(weight) hduList = fits.HDUList([hduData, hduWeight]) hduList.writeto(r, overwrite=True) hdu.close() try: if fits.getval(r, 'NORM') == 'N': fits.setval(r, 'NORM', value='Y') normalize(r) except KeyError: fits.setval(r, 'NORM', value='Y') normalize(r) #fill all masked values with zero zeros_mask(location) #make lists of residuals and their weight maps residual_list = "%s/residuals/inputs.txt" % (location) weight_list = "%s/templates/weights.txt" % (location) with open(residual_list, 'w+') as i: for r in residuals: i.write("%s[0]\n" % (r)) with open(weight_list, 'w+') as w: for r in residuals: w.write("%s[1]\n" % (r)) #customize swarp config file for MR initialize.create_configs(location) test_data = fits.getdata(residuals[0]) correct_shape = test_data.shape config_loc = "%s/configs/default.swarp.MR" % (location) with open(config_loc, 'r') as conf: lines = conf.readlines() lines[ 4] = "IMAGEOUT_NAME" + " " + "%s/residuals/MR.fits" % ( location) + "\n" lines[ 5] = "WEIGHTOUT_NAME" + " " + "%s/residuals/MR_swarp_weight.fits" % ( location) + "\n" lines[ 15] = "WEIGHT_IMAGE" + " " + "@%s/templates/weights.txt" % ( location) + "\n" lines[ 38] = "IMAGE_SIZE" + " " + "%s, %s" % correct_shape[:: -1] + "\n" with open(config_loc, 'w') as conf: conf.writelines(lines) #perform image combination try: os.system("swarp @%s/residuals/inputs.txt -c %s" % (location, config_loc)) #turn MR output weight map into binary weight map and add it as #the first extension to the final MR.fits file MR_data = fits.getdata("%s/residuals/MR.fits" % (location)) MR_header = fits.getheader("%s/residuals/MR.fits" % (location)) MR_mask = fits.getdata("%s/residuals/MR_swarp_weight.fits" % (location)) mask_median = np.median(MR_mask) mask_std = np.std(MR_mask) threshold = mask_median - (mask_std) MR_mask[MR_mask < threshold] = 0 MR_mask[MR_mask >= threshold] = 1 MR_hdu = fits.PrimaryHDU(MR_data, header=MR_header) MR_hdu_mask = fits.ImageHDU(MR_mask) MR_hdu_list = fits.HDUList([MR_hdu, MR_hdu_mask]) MR_hdu_list.writeto("%s/residuals/MR.fits" % (location), overwrite=True) try: os.remove("%s/residuals/MR_swarp_weight.fits" % (location)) except: print("\-> Error: Could not remove MR weight map\n") except: print("-> Error: Master residual construction failed\n") else: print("-> Error: Problem with number of residuals\n")
def sim_sameField(location, mode='moffat', numIms=100, bkg_mag=22.5, fwhm_min=3, fwhm_max=6, rot_min=-2.5, rot_max=2.5, shift_min=-2, shift_max=2, scale_mult=(0,1.5), scale_add=(-20,50), zero_point=25): '''Test **OASIS**'s ability to handle frame-by-frame variations in astronomical data and filter out false-positive sources. The procedure of the simulation is as follows: 1. Copies a random science image from the specified dataset to the **simulations** directory. 2. A source catalog of the chosen science image is made, containing information on each source's centroid location and total flux. 3. Using this source catalog, simulations of the chosen science image are made, all with constant source flux and location, but with different backgrounds, seeing, and pointing. 4. The set of simulated images are sent through the **OASIS Pipeline**. 5. Low numbers of detected sources signifies a successful simulation. There are no variable objects in the simulated images, so ideally zero sources should be detected by **OASIS**. :param str location: Path of data file tree (contains the **configs**, **data**, **psf**, **residuals**, **sources**, **templates** directories). Use a comma-separated list for mapping to multiple datasets. :param str mode: Simulation mode. Method by which simulated images are made. All images are given a uniform background, then smeared according to Poisson statistics. * *moffat* (default): Sources are convolved with a 2D Moffat kernel. * *gauss*: Sources are convolved with a symmetric 2D Gaussian kernel. * *real*: The actual PSF model of the chosen science image is used as the convolution kernel. * *sky*: AstrOmatic program ``SkyMaker`` (Bertin) is used to make simulated images. :param int numIms (default=100): Number of simulated images to make. :param float bkg_mag: Average background level in mags. Actual simulated background levels are chosen to be a random value within the interval :math:`[bkg\_mag-1.5, bkg\_mag+1.5]`. :param float fwhm_min: Minimum FWHM of simulated images in pixels. :param float fwhm_max: Maximum FWHM of simulated images in pixels. :param float rot_min: Lower bound on angle of rotation in degrees. :param float rot_max: Upper bound on angle of rotation in degrees. :param float shift_min: Lower bound on (X,Y) shift in pixels. :param float shift_max: Upper bound on (X,Y) shift in pixels. :param tuple scale_mult: Interval of acceptable multiplicative scale factors. :param tuple scale_add: Interval of acceptable additive scale factors. :param float zero_point: Zero point magnitude. :returns: Standard **OASIS Pipeline** output, residual frames located in **residuals** and source catalogs located in **sources**. ''' ref_im = glob.glob(location + '/data/*_ref_A_.fits') if os.path.exists(location) == False: print("-> Error: Problem with path name(s)-- make sure paths exist and are entered correctly\n-> Exiting...") sys.exit() if len(ref_im) != 1: print("-> Error: Problem with number of reference images\n-> Exiting...\n") sys.exit() ref_im = ref_im[0] ref_fwhm = fwhm(ref_im) path_splits = ref_im.split('/') image_name = path_splits[-1] sim_loc = location.replace('targets', 'simulations') len_loc = len(loc.split('/')) tar = path_splits[len_loc+2] copy_to_sim(tar, image=ref_im, mode='samefield') ref_psf = glob.glob("%s/psf/*_ref_A_.psf" % (sim_loc)) if len(ref_psf) != 1: print("-> Error: Problem with number of reference PSF files\n-> Exiting...\n") sys.exit() try: clear_contents(sim_loc) except: pass images = glob.glob("%s/data/*.fits" % (sim_loc)) ref_im_sim = ref_im.replace("targets", "simulations") #delete all original images except reference for i in images: name = i.split('/')[-1] if name != image_name: os.remove(i) #create configs directory if none exists create_configs(sim_loc) #make source catalog of reference using SExtractor sim_config = "%s/configs/default_sim.sex" % (sim_loc) sim_params = "%s/configs/default_param_sim.sex" % (sim_loc) with open(sim_config, 'r') as conf: lines = conf.readlines() lines[6] = "CATALOG_NAME" + " " + "%s/data/reference.cat" % (sim_loc) + "\n" lines[9] = "PARAMETERS_NAME" + " " + sim_params + "\n" lines[22] = "FILTER_NAME" + " " + "%s/configs/default.conv" % (sim_loc) + "\n" lines[70] = "SEEING_FWHM" + " " + str(ref_fwhm) + "\n" lines[127] = "PSF_NAME" + " " + ref_psf[0] + "\n" with open(sim_config, 'w') as conf_write: conf_write.writelines(lines) os.system("sextractor %s[0] -c %s" % (ref_im_sim, sim_config)) #extract x_pos, y_pos, and fluxes from SExtractor catalog ref_cat = "%s/data/reference.cat" % (sim_loc) with open(ref_cat, 'r') as cat: cat_lines = cat.readlines() #get simulated image's metadata ref_hdu = fits.open(ref_im_sim) ref_data = ref_hdu[0].data ref_header = ref_hdu[0].header ref_mask = ref_hdu[1].data try: weight_check = fits.getval(ref_im_sim, 'WEIGHT') except: weight_check = 'N' if weight_check == 'Y': ref_mask = (ref_mask-1)*-1 ref_mask = ref_mask.astype(np.int64) ref_hdu.close() from astropy.stats import sigma_clipped_stats mean, median, std = sigma_clipped_stats(ref_data, sigma=3.0) #extract simulated image's source information from SExtractor catalog x_pos = [] y_pos = [] flux = [] sources = {} for c in cat_lines: splits = c.split() if splits[0] != '#': flux.append(float(splits[0])) x_pos.append(round(float(splits[3]))) y_pos.append(round(float(splits[4]))) sources.update({float(splits[0]) : (round(float(splits[3])), round(float(splits[4])))}) flux_ordered = sorted(sources) flux_iter = round(len(flux)*0.99) flux_sim = flux_ordered[flux_iter] xy_sim = sources[flux_sim] #if mode is set to use SkyMaker for making the simulations, configure SkyMaker if mode == 'sky': mags = [] for f in flux: mags.append((28-(np.log(f)))) with open("%s/configs/sky_list.txt" % (sim_loc), "w+") as sky_list: for i in range(len(flux)): sky_list.write("100 %.3f %.3f %.3f\n" % (x_pos[i], y_pos[i], mags[i])) #get pixel scale of reference image pixscale = float(ref_header['PIXSCALE']) #define oversampling oversample = pixscale*25 #define sky.config location sky_config = "%s/configs/sky.config" % (sim_loc) #start making fake images print("\n-> Making simulated images...") for n in tqdm(range(numIms)): #define image name if n == 0: image_name = '%s/data/%d_ref_A_.fits' % (sim_loc, n) else: image_name = '%s/data/%d_N_.fits' % (sim_loc, n) #for each image: make sources w/ random fwhm b/w (3,6), rotate/zoom, shift, add a different gaussian dist. of noise, change scale linearly, poisson smear #define FWHM of simulation image_fwhm = ((fwhm_max-fwhm_min) * np.random.random()) + fwhm_min #based on the mode chosen, create the corresponding convolution kernel and make simulated image if mode != 'sky': if mode == 'moffat': moffat_kernel_1 = Moffat2DKernel(gamma=make_stars.get_moffat_gamma(image_fwhm), alpha=7) moffat_kernel_2 = Moffat2DKernel(gamma=make_stars.get_moffat_gamma(image_fwhm), alpha=2) conv_kernel = (0.8*moffat_kernel_1) + (0.2*moffat_kernel_2) elif mode == 'gauss': gaussian_kernel_1 = Gaussian2DKernel(x_stddev=(image_fwhm/2.355), y_stddev=(image_fwhm/2.355)) gaussian_kernel_2 = Gaussian2DKernel(x_stddev=((image_fwhm*2)/2.355), y_stddev=((image_fwhm*2)/2.355)) conv_kernel = (0.9*gaussian_kernel_1) + (0.1*gaussian_kernel_2) elif mode == 'real': conv_kernel = get_first_model(ref_im) try: conv_kernel /= np.sum(conv_kernel) except: pass flux_variable = np.array(flux) * np.random.random() * 2 image = make_stars.make_image(ref_data.shape[0], ref_data.shape[1], x_loc=y_pos, y_loc=x_pos, fluxes=flux_variable, psf=[conv_kernel]) #if mode is set to 'sky' use SkyMaker to make simulated image elif mode == 'sky': bkg_Mag = (1.5*np.random.random()) + bkg_mag image_fwhm_arcsec = image_fwhm*pixscale with open(sky_config, 'r') as sky: sky_lines = sky.readlines() sky_lines[6] = "IMAGE_NAME" + " " + image_name + "\n" sky_lines[7] = "IMAGE_SIZE" + " " + str("%d, %d" % (ref_data.shape[1], ref_data.shape[0])) + "\n" sky_lines[19] = "SATUR_LEVEL" + " " + str(ref_header['SATURATE']) + "\n" sky_lines[21] = "EXPOSURE_TIME" + " " + str(ref_header['EXPTIME']) + "\n" sky_lines[26] = "PIXEL_SIZE" + " " + str(pixscale) + "\n" sky_lines[34] = "SEEING_FWHM" + " " + str(image_fwhm_arcsec) + "\n" sky_lines[37] = "PSF_OVERSAMP" + " " + str(oversample) + "\n" sky_lines[65] = "BACK_MAG" + " " + str(bkg_Mag) + "\n" with open(sky_config, 'w') as sky: sky.writelines(sky_lines) os.system("sky %s/configs/sky_list.txt -c %s" % (sim_loc, sky_config)) try: os.remove("%s/data/%s.list" % (sim_loc, image_name[:-5])) except: pass image = fits.getdata(image_name) else: print("-> Error: Please enter a valid mode (gauss, moffat, sky, real)\n-> Exiting...") sys.exit() #now we start the warping of each simulation #first rotate/zoom (angle is random b/w 0 and 30 degrees, zoom is random b/w 0 and 2) if n != 0: #define initial mask for each simulation Mask = np.zeros(image.shape) rot_angle = ((rot_max-rot_min)*np.random.random())+rot_min dx = (shift_max-shift_min) * np.random.random() - shift_min dy = (shift_max-shift_min) * np.random.random() - shift_min image = rotate(image, rot_angle, reshape=False) image = shift(image, [dx,dy]) Mask = rotate(ref_mask, rot_angle, reshape=False, cval=1) Mask = shift(Mask, [dx,dy], cval=1) else: Mask = ref_mask #for non-SkyMaker simulations, add in a random background, poisson smear the image, and rescale it if mode != 'sky': #add constant background bkg_loc = 2.512**(zero_point - bkg_mag) bkg_scl = ((std+5)-(std-5))*np.random.random()+(std-5) bkg = np.random.normal(loc=bkg_loc, scale=bkg_scl, size=image.shape) image = np.add(image, bkg) #poisson smear negative_image = np.zeros(image.shape) negative_image[:] = image[:] image[image < 0] = 0 negative_image[negative_image > 0] = 0 image = np.random.poisson(image) image = image.astype(np.float64) negative_image *= -1 negative_image = np.random.poisson(negative_image) negative_image = negative_image.astype(np.float64) negative_image *= -1 image += negative_image #rescale image linearly a = ((scale_mult[1] - scale_mult[0])*np.random.random()) + scale_mult[0] b = (scale_add[1] - scale_add[0])*np.random.random() - scale_add[0] image *= a image += b #write new image to data folder in target's simulations folder newHDUData = fits.PrimaryHDU(image, header=ref_header) newHDUMask = fits.ImageHDU(Mask) newHDUList = fits.HDUList([newHDUData, newHDUMask]) newHDUList.writeto(image_name, overwrite=True) newHDU = fits.open(image_name, mode='update') (newHDU[0].header).set('WEIGHT', 'N') (newHDU[0].header).set('SCALED', 'N') newHDU.close() os.system("mv %s %s" % (ref_im_sim, sim_loc)) os.system("mv %s %s" % (ref_psf, sim_loc)) os.system("mv %s %s.cat" % (ref_psf[:-4], sim_loc)) if mode == 'sky': sim_lists = glob.glob("%s/data/*.list" % (sim_loc)) for sl in sim_lists: os.remove(sl) pipeline.pipeline_run_sim(sim_loc, sim=False) print(flux_iter, flux_sim, xy_sim)
def sextractor(location): ''' runs SExtractor on all residual images ''' sources = location + "/sources" residuals = location + "/residuals" check = os.path.exists(sources) check_temp = os.path.exists(sources + '/temp') length = len(residuals) + 1 if check == False: os.system("mkdir %s" % (sources)) os.system("mkdir %s/temp" % (sources)) else: if check_temp == False: os.system("mkdir %s/temp" % (sources)) images = glob.glob(residuals + "/*_residual_.fits") initialize.create_configs(location) config_loc = location + '/configs/default.sex' with open(config_loc, 'r') as config: data = config.readlines() config.close() data[ 9] = "PARAMETERS_NAME" + " " + location + "/configs/default.param" + "\n" data[ 20] = "FILTER_NAME" + " " + location + "/configs/default.conv" + "\n" with open(config_loc, 'w') as config: config.writelines(data) config.close() print("-> Converting all residual masks into weight maps...\n") for r in tqdm(images): weight = weight_map(r) hdu = fits.open(r, mode='update') data = hdu[0].data hdr = hdu[0].header try: if hdr['WEIGHT'] == 'N': hdr.set('WEIGHT', 'Y') hduData = fits.PrimaryHDU(data, header=hdr) hduWeight = fits.ImageHDU(weight) hduList = fits.HDUList([hduData, hduWeight]) hduList.writeto(r, overwrite=True) except KeyError: hdr.set('WEIGHT', 'Y') hduData = fits.PrimaryHDU(data, header=hdr) hduWeight = fits.ImageHDU(weight) hduList = fits.HDUList([hduData, hduWeight]) hduList.writeto(r, overwrite=True) hdu.close() try: if fits.getval(r, 'NORM') == 'N': fits.setval(r, 'NORM', value='Y') MR.normalize(r) except KeyError: fits.setval(r, 'NORM', value='Y') MR.normalize(r) print("\n-> SExtracting residual images...") for i in tqdm(images): if np.std(fits.getdata(i)) != 0: name = i[length:-5] data_name = location + '/data/' + name.replace('residual_', '') + '.fits' FWHM = psf.fwhm(data_name) im_hdu = fits.open(data_name) im_header = im_hdu[0].header saturate = im_header['SATURATE'] pixscale = im_header['PIXSCALE'] im_hdu.close() with open(config_loc, 'r') as config: data = config.readlines() config.close() data[51] = "SATUR_LEVEL" + " " + str(saturate) + "\n" data[62] = "SEEING_FWHM" + " " + str(FWHM) + "\n" data[ 106] = "PSF_NAME" + " " + location + "/psf/" + name[: -9] + ".psf" + "\n" data[58] = "PIXEL_SCALE" + " " + str(pixscale) + "\n" data[32] = "WEIGHT_IMAGE" + " " + "%s[1]" % (i) + "\n" with open(config_loc, 'w') as config: config.writelines(data) config.close() os.system("sextractor %s[0]> %s/temp/%s.txt -c %s" % (i, sources, name, config_loc)) temp_hdu_data = fits.PrimaryHDU((fits.getdata(i)) * -1, header=fits.getheader(i)) temp_hdu_mask = fits.ImageHDU(fits.getdata(i, 1)) temp_hdu_list = fits.HDUList([temp_hdu_data, temp_hdu_mask]) temp_hdu_list.writeto("%s/residuals/temp.fits" % (location)) os.system( "sextractor %s/residuals/temp.fits[0]> %s/temp/%s_2.txt -c %s" % (location, sources, name, config_loc)) append_negative_sources(i) os.remove("%s/residuals/temp.fits" % (location)) else: name = i[length:-5] with open("%s/temp/%s.txt" % (sources, name), 'w') as bad_res_cat: bad_res_cat.write("# Bad residual, did not SExtract\n") print( "-> SExtracted %d images, catalogues placed in 'sources' directory\n" % (len(images))) print("-> Filtering source catalogs...\n") src_join(location) filter_sources(location)
def sextractor_psf(location): x = 0 psf_loc = location + "/psf" data = location + "/data" templates = location + "/templates" check = os.path.exists(psf_loc) if check == False: os.system("mkdir %s" % (psf_loc)) temps = glob.glob(templates + "/*.fits") images = glob.glob(data + "/*_A_.fits") for t in temps: images.append(t) cats = glob.glob(location + '/psf/*.cat') images_names = [(i.split('/')[-1])[:-5] for i in images] cats_names = [(c.split('/')[-1])[:-4] for c in cats] imageCats = [im for im in images_names if im not in cats_names] images = [] if temps == []: temps.append('') for imcats in imageCats: if imcats == (temps[0].split('/')[-1])[:-5]: images.append(temps[0]) else: images.append(location + '/data/' + imcats + '.fits') initialize.create_configs(location) config_loc = location + '/configs/psf.sex' with open(config_loc, 'r') as config: data = config.readlines() config.close() data[ 9] = "PARAMETERS_NAME" + " " + location + "/configs/default.psfex" + "\n" data[ 19] = "FILTER_NAME" + " " + location + "/configs/default.conv" + "\n" with open(config_loc, 'w') as config: config.writelines(data) config.close() print("\n-> Creating PSF catalogs...") if len(temps) == 1: for i in tqdm(images): name = i.split('/')[-1][:-5] hdu = fits.open(i) hdr = hdu[0].header pixscale = hdr['PIXSCALE'] hdu.close() with open(config_loc, 'r') as config: data = config.readlines() config.close() data[ 6] = "CATALOG_NAME" + " " + psf_loc + "/" + name + ".cat" + "\n" data[44] = "PIXEL_SCALE" + " " + str(pixscale) + "\n" with open(config_loc, 'w') as config: config.writelines(data) config.close() os.system("sextractor %s[0] -c %s" % (i, config_loc)) # x += 1 # per = float(x)/float(len(images)) * 100 # print("\t %.1f%% sextracted..." % (per)) print( "-> SExtracted %d images, catalogues placed in 'psf' directory\n" % (len(images))) else: print("\n-> Error: Problem with number of template images\n") sys.exit() return images
def isis_sub(location): x = 0 images = glob.glob(location + "/data/*_A_.fits") template = glob.glob(location + "/templates/*.fits") residuals = glob.glob(location + "/residuals/*residual_.fits") images_names = [(i.split('/')[-1])[:-5] for i in images] res_names = [(r.split('/')[-1])[:-14] for r in residuals] resids = [res for res in images_names if res not in res_names] ims = [] for rs in resids: ims.append(location + '/data/' + rs + '.fits') if ims != []: if len(template) == 1: ais_loc = os.path.dirname( initialize.__file__) + "/AIS/package/bin/./mrj_phot" initialize.create_configs(location) ais_config_loc = location + '/configs/default_config' cwd = os.getcwd() psf_data = glob.glob(location + '/psf/*') template_mask = fits.getdata(template[0], 1) if len(psf_data) == 2 * (len(images) + 1): try: os.mkdir(cwd + "/AIS_temp") except FileExistsError: pass os.chdir(cwd + "/AIS_temp") length = len(location) + 5 print("\n-> Subtracting images...") for i in ims: int_match_to_template(location, i, template[0]) os.system(ais_loc + " " + i + " " + template[0] + " -c " + ais_config_loc) os.system( "mv -f %s/AIS_temp/conv.fits %s/residuals/%sresidual_.fits" % (cwd, location, i[length:-5])) hdu = fits.open(location + '/residuals/' + i[length:-5] + 'residual_.fits', mode='update') hdr = hdu[0].header hdr.set('OPTIMIZE', 'N') hdu.close() image_hdu = fits.open( (i.replace('residual_', '')).replace('residuals', 'data')) image_hduMask = np.logical_or( np.logical_not(image_hdu[1].data), np.logical_not(template_mask)).astype(int) image_hdu.close() hdu = fits.open(location + '/residuals/' + i[length:-5] + 'residual_.fits') data = hdu[0].data hdr = hdu[0].header hdu.close() hduData = fits.PrimaryHDU(data, header=hdr) hduMask = fits.ImageHDU(image_hduMask) hduList = fits.HDUList([hduData, hduMask]) hduList.writeto(location + '/residuals/' + i[length:-5] + 'residual_.fits', overwrite=True) x += 1 per = float(x) / float(len(ims)) * 100 print("\t %.1f%% subtracted..." % (per)) else: print( "-> Error: Need PSFs before running subtraction\n-> Run psf.py first" ) print( "-> If any images have been manually removed from the data directory, delete all contents of the psf directory and run OasisPy again\n" ) sys.exit() else: print("-> Subtraction failure: Template missing") sys.exit() os.chdir(cwd) shutil.rmtree(cwd + "/AIS_temp") else: print("-> Images have already been subtracted")