def make_mock_image(self, truex, truey, b, fluxes, nc, cf, noise, two_star_mode=None): mocks = [] if two_star_mode is not None: mock0 = image_model_eval(truex, truey, fluxes[b], self.truebacks[b], self.imsz, nc, cf) else: mock0 = image_model_eval(truex, truey, fluxes[b], self.normal_backs[b], self.imsz, nc, cf) mock0[mock0 < 1] = 1. variance = mock0 / self.gain print np.mean(variance), np.std(variance) for n in xrange(self.n_realizations): # here is where I would put an if statement like # if two_star_mode == 'rx3': # mock = mock0 + (np.sqrt(variance)*np.mean(noise[:,n])) # i believe it would be the mean of the three since we already coadd the model image # from which we calculate the variance. mock = mock0 + (np.sqrt(variance) * noise[b, n]) mocks.append(mock) return mocks
def make_srcmap(self, ifield, cat, flux_idx=-1, band=0, nbin=0., nx=1024, ny=1024, nwide=17, multfac=7.0, \ pcat_model_eval=False, libmmult=None, dx=2.5, dy=-1.0): ''' This function takes in a catalog, finds the PSF for the specific ifield, makes a PSF template and then populates an image with model sources. When we use galaxies for mocks we can do this because CIBER's angular resolution is large enough that galaxies are well modeled as point sources. Note (5/21/20): This function currently only places sources at integer locations, which should probably change for any real implementation used in analysis. Note (5/22/20): with the integration of PCAT's model evaluation routines, we can now do sub-pixel source placement and it is a factor of 6 faster than the original implementation when at scale (e.g. catalog sources down to 25th magnitude) dx and dy are offsets meant to be used when comparing PCAT's model evaluation with the original, but is not strictly necessary for actual model evaluation. In other words, these can be set to zero when not doing comparisons. ''' Nsrc = cat.shape[0] srcmap = np.zeros((nx * 2, ny * 2)) Nlarge = nx + 30 + 30 if self.psf_template is None: print('generating psf template because it was none') self.get_psf(ifield=ifield, band=band, nx=nx, ny=ny, multfac=multfac, poly_fit=pcat_model_eval, nwide=nwide) if pcat_model_eval: srcmap = image_model_eval( np.array(cat[:, 0]).astype(np.float32) + dx, np.array(cat[:, 1]).astype(np.float32) - dy, np.array(cat[:, flux_idx]).astype(np.float32), 0., (nx, ny), 35, self.cf, lib=self.libmmult.pcat_model_eval) return srcmap else: xs = np.round(cat[:, 0]).astype(np.int32) ys = np.round(cat[:, 1]).astype(np.int32) for i in range(Nsrc): srcmap[Nlarge / 2 + 2 + xs[i] - nwide:Nlarge / 2 + 2 + xs[i] + nwide + 1, Nlarge / 2 - 1 + ys[i] - nwide:Nlarge / 2 - 1 + ys[i] + nwide + 1] += self.psf_template * cat[i, flux_idx] return srcmap[nx / 2 + 30:3 * nx / 2 + 30, ny / 2 + 30:3 * ny / 2 + 30]
array_2d_double = npct.ndpointer(dtype=np.float64, ndim=2, flags="C_CONTIGUOUS") libmmult = npct.load_library('pcat-lion', '.') libmmult.pcat_model_eval.restype = None libmmult.pcat_model_eval.argtypes = [ c_int, c_int, c_int, c_int, c_int, array_2d_float, array_2d_float, array_2d_float, array_1d_int, array_1d_int, array_2d_float, array_2d_float, array_2d_float, array_2d_double, c_int, c_int, c_int, c_int ] noise = np.random.normal(size=(imsz[1], imsz[0])).astype(np.float32) mock = image_model_eval(np.concatenate([truex, xphon]), np.concatenate([truey, yphon]), np.concatenate([truef, fphon]), trueback, imsz, nc, cf, lib=libmmult.pcat_model_eval) mock[mock < 1] = 1. # maybe some negative pixels variance = mock / gain oldmock = mock.copy() mock += (np.sqrt(variance) * np.random.normal(size=(imsz[1], imsz[0]))).astype( np.float32) diff = mock - oldmock print np.sum(diff * diff / variance) f = open('Data/mockL_pix.txt', 'w') f.write('%1d\t%1d\t1\n0.\t%0.3f' % (imsz[0], imsz[1], gain)) f.close()
rhol = np.sqrt(nstar / float(max_x * max_y)) nx = np.ceil(max_x * rhol) ny = np.ceil(max_y * rhol) lx = max_x / float(nx) ly = max_y / float(ny) truex = np.tile(np.arange(nx) * lx + lx / 2. + minx, (ny, 1)).flatten().astype(np.float32) truey = np.tile(np.arange(ny) * ly + ly / 2. + minx, (nx, 1)).flatten('F').astype(np.float32) ind = np.random.choice(int(nx * ny), size=nstar, replace=False) truef = np.ones_like(truex) * 20000 print "Max flux %1.3e" % (max(truef)) noise = np.random.normal(size=(imsz[1], imsz[0])).astype(np.float32) mock = image_model_eval(truex, truey, truef, trueback, imsz, nc, cf) mock[mock < 1] = 1. # maybe some negative pixels variance = trueback / gain # mock/gain mock += (np.sqrt(variance) * np.random.normal(size=(imsz[1], imsz[0]))).astype( np.float32) plt.imshow(mock, origin='lower', interpolation='None') plt.show() fname = 'Data/mock' + flabel f = open(fname + 'pix.txt', 'w') f.write('%1d\t%1d\t1\n0.\t%0.3f' % (imsz[0], imsz[1], gain)) f.close() np.savetxt(fname + 'cts.txt', mock)
def multiband_retro_frames(result_path, ref_cat_path, data_path,\ hubble_cat_path=None, \ chain_datatype='npz', \ nframes=10, \ mock2_type=None, \ boolplotsave=1, \ boolplotshow=0, \ frame_number_list=None, \ datatype='real', \ plttype='pdf', \ bright_n=50, \ imdim=100, \ bands=['r']): if datatype == 'mock': labldata = 'Mock Truth' else: labldata = datatype sizefac = 10. * 136 if chain_datatype.lower() == 'npz': chain = np.load(chain_path + '/chain.npz') elif chain_datatype.lower() == 'hdf5': chain = h5py.File(chain_path + '/chain.hdf5', 'r') else: raise IOError( 'Could not read in data type, please use .npz or .hdf5 files.') nsrcs = chain['n'] xsrcs = np.array(chain['x'], dtype=np.float32) ysrcs = np.array(chain['y'], dtype=np.float32) fsrcs = chain['f'] colorsrcs = chain['colors'] chi2 = chain['chi2'] timestats = chain['times'] # should be an array of shape (nsamp, 5) acceptfracs = chain['accept'] nmgy_per_count = chain['nmgy'] bkgs = chain['back'] pixel_transfer_mats = chain['pixel_transfer_mats'] print 'backgrounds:', bkgs nsamp = len(nsrcs) nbands = len(bands) ref_cat = np.loadtxt(ref_cat_path) ref_x = ref_cat[:, 0] ref_y = ref_cat[:, 1] ref_f = ref_cat[:, 2:] ref_mags = [] for b in xrange(nbands): ref_mag = adu_to_magnitude(ref_f[:, b], nmgy_per_count[b]) ref_mags.append(ref_mag) ref_colors = [] for b in xrange(nbands - 1): ref_color = ref_mags[b] - ref_mags[b + 1] ref_colors.append(ref_color) hpath2 = '/n/fink1/rfeder/mpcat/multiband_pcat/Data/idR-002583-2-0136/hubble_catalog_2583-2-0136_astrans.txt' # use transform q to get hubble coordinates in other bands, or mean_dpos if hubble_cat_path is not None: hubble_coords, hf, hmask, posmask, hx, hy = get_hubble(hubble_cat_path, hpath_2=hpath2) #hubble_coords = np.loadtxt(hubble_cat_path) #hx0 = hubble_coords[:,0]-310 #hy0 = hubble_coords[:,1]-630 #h_rmag = hubble_coords[:,2] #hx1, hy1 = transform_q(h_coords[0][posmask], h_coords[1][posmask], pixel_transfer_mats[first_frame_band_no- psf_basic_path = data_path + '/' + dataname + '/psfs/' + dataname + '-psf.txt' cts_basic_path = data_path + '/' + dataname + '/cts/' + dataname + '-cts.txt' gains = dict() biases = dict() #if datatype != 'mock': # frame_basic_path = data_path+'/'+dataname+'/frames/frame--'+run_cam_field+'.fits' # gains = dict({"r":4.61999, "i":4.38999, "g":4.2, "z":5.67999}) # biases = dict({"r":1044., "i":1177., "g":1113., "z":1060.}) for band in bands: pix_path = data_path + '/' + dataname + '/pixs/' + dataname + '-pix' + band + '.txt' g = open(pix_path) a = g.readline().split() bias, gain = [np.float32(i) for i in np.float32(g.readline().split())] gains[band] = gain biases[band] = bias #print 'biases:', biases #print 'gains:', gains #print 'nmgy_per_count:', nmgy_per_count color_bins = np.linspace(-4, 4, 50) #for band in bands: #if datatype=='mock': #pix_path = data_path+'/'+dataname+'/pixs/'+dataname+'-pix'+band+'.txt' #g = open(pix_path) #a = g.readline().split() #np.float32(g.readline().split()) #frame_path = frame_basic_path.replace('--', '-'+band+'-') #mean_dpos = dict({'r-i':[-1.,3.], 'r-g':[3.,10.], 'r-z':[1.,7.], 'r-r':[0.,0.]}) mean_dpos = dict({'r-i': [-1., 3.], 'r-g': [-2., 12.]}) if frame_number_list is None: frame_number_list = np.linspace(0, nsamp, nframes) c = 0 for num in frame_number_list: num = max(int(num - 1), 0) x = 5 * nbands # data, residual, magnitude distribution plt.figure(figsize=(15, 10)) for b in xrange(nbands): bop = int(3 * b) psf_path = psf_basic_path.replace('.txt', bands[b] + '.txt') #if datatype != 'mock': # psf_path = psf_path.replace('.txt', '-refit_g.txt') # only if using refit psf! #print 'psf_path:', psf_path cts_path = cts_basic_path.replace('.txt', bands[b] + '.txt') psf, nc, cf = get_psf_and_vals(psf_path) #get background data = np.loadtxt(cts_path) imsz = [len(data), len(data[0])] if datatype != 'mock': data -= biases[bands[b]] #print biases[bands[b]] if b == 0: xs = xsrcs[num] ys = ysrcs[num] if hubble_cat_path is not None: hx_band, hy_band = hx, hy else: xs, ys = transform_q(xsrcs[num], ysrcs[num], pixel_transfer_mats[b - 1]) if datatype != 'mock': col_name = bands[0] + '-' + bands[b] #print mean_dpos[col_name] xs -= mean_dpos[col_name][0] ys -= mean_dpos[col_name][1] if hubble_cat_path is not None: #print 'pixel_transfer_mats' #print pixel_transfer_mats hx_band, hy_band = transform_q( hx, hy, pixel_transfer_mats[b - 1]) hx_band -= mean_dpos[col_name][0] hy_band -= mean_dpos[col_name][1] model = image_model_eval(xs, ys, fsrcs[b, num], bkgs[b], imsz, nc, cf) resid = data - model variance = data / gains[bands[b]] weight = 1. / variance np.savetxt( result_path + '/residuals/resid_' + str(num) + str(bands[b]) + '.txt', resid * np.sqrt(weight)) # plt.figure() # plt.imshow(resids[0,b,:,:]*np.sqrt(weight), origin='lower', interpolation='none', cmap='Greys', vmin=-30, vmax=30) # if boolplotsave: # plt.savefig(result_path + '/residuals/resid_' + str(num) +str(bands[b])+ '.'+plttype, bbox_inches='tight') # plt.close() #plt.figure(figsize=(15,x)) plt.subplot(nbands, 3, 1 + bop) plt.imshow(data, origin='lower', interpolation='none', cmap='Greys', vmin=np.min(data), vmax=np.percentile(data, 95)) plt.colorbar() if hubble_cat_path is not None: #print 'using Hubble coordinates' #print 'hx_band[posmask]:' #print hx_band[posmask] #print 'hmask length' #print len(hmask) #print 'hx_band[posmask][hmask]' #print hx_band[posmask][hmask] plt.scatter(hx_band[hmask], hy_band[hmask], marker='+', s=2 * mag_to_cts(hf[hmask], nmgy_per_count[b]) / sizefac, color='lime') #plt.scatter(hubble_coords[1+2*b][posmask][hmask], hubble_coords[2*b][posmask][hmask], marker='+', s=2*mag_to_cts(hf[hmask], nmgy) / sizefac, color='lime') #hubble else: mask = ref_f[:, 0] > 250 # will have to change this for other data sets plt.scatter(ref_x[mask], ref_y[mask], marker='+', s=ref_f[:, b][mask] / sizefac, color='lime') mask = np.logical_not(mask) plt.scatter(ref_x[mask], ref_y[mask], marker='+', s=ref_f[:, b][mask] / sizefac, color='g') if len(data) > 100: plt.scatter(xs, ys, marker='x', s=(len(data)**2 / 1e5) * fsrcs[b, num] / (2 * sizefac), color='r') else: plt.scatter(xs, ys, marker='x', s=(1e4 / len(data)**2) * fsrcs[b, num] / (2 * sizefac), color='r') plt.xlim(-0.5, len(data) - 0.5) plt.ylim(-0.5, len(data) - 0.5) plt.subplot(nbands, 3, 2 + bop) plt.imshow(resid * np.sqrt(weight), origin='lower', interpolation='none', cmap='Greys', vmin=-5, vmax=5) plt.xlim(-0.5, len(data) - 0.5) plt.ylim(-0.5, len(data) - 0.5) plt.colorbar() plt.subplot(nbands, 3, 3 + bop) mags = adu_to_magnitude(fsrcs[b, num], nmgy_per_count[b]) mags = mags[~np.isinf(mags)] (n, bins, patches) = plt.hist(ref_mags[b], histtype='step', label=labldata, color='g') plt.hist(mags, bins=bins, alpha=0.5, label='Chain - ' + bands[b], color='r', histtype='step') plt.legend() plt.xlabel(str(bands[b])) plt.yscale('log') if boolplotsave: plt.savefig(result_path + '/frames/sample_' + str(num) + '_mags.' + plttype, bbox_inches='tight') if boolplotshow: plt.show() plt.close() plt.figure(figsize=(10, 5)) for b in xrange(nbands - 1): colors = colorsrcs[b][num] colors = adus_to_color(fsrcs[0, num], fsrcs[b + 1, num], nmgy_per_count) plt.subplot(1, nbands - 1, b + 1) #print 'colors:', colors plt.hist(colors[~np.isnan(colors)], label='Chain', bins=color_bins, bottom=0.1, histtype='step', color='r') plt.hist(ref_colors[b], label=labldata, bins=color_bins, bottom=0.1, histtype='step', color='g') plt.legend(loc=2) plt.yscale('log') plt.xlabel(bands[0] + ' - ' + bands[b + 1]) if boolplotsave and nbands > 1: plt.savefig(result_path + '/color_histogram/color_histograms_sample_' + str(num) + '.' + plttype, bbox_inches='tight') if boolplotshow and nbands > 1: plt.show() plt.close()
def writ_truedata(): pathlion = os.environ["LION_PATH"] sys.path.insert(0, pathlion) from image_eval import image_model_eval, psf_poly_fit dictglob = dict() sersindx = 2. numbsidegrid = 20 pathliondata = os.environ["LION_DATA_PATH"] + '/data/' pathlionimag = os.environ["LION_DATA_PATH"] + '/imag/' fileobjt = open(pathliondata + 'sdss.0921_psf.txt') numbsidepsfn, factsamp = [np.int32(i) for i in fileobjt.readline().split()] fileobjt.close() psfn = np.loadtxt(pathliondata + 'sdss.0921_psf.txt', skiprows=1).astype(np.float32) cpsf = psf_poly_fit(psfn, factsamp) cpsf = cpsf.reshape((-1, cpsf.shape[2])) np.random.seed(0) numbside = [100, 100] # generate stars numbstar = 1 fluxdistslop = np.float32(2.0) minmflux = np.float32(250.) logtflux = np.random.exponential(scale=1. / (fluxdistslop - 1.), size=numbstar).astype(np.float32) dictglob['numbstar'] = numbstar dictglob['xposstar'] = (np.random.uniform(size=numbstar) * (numbside[0] - 1)).astype(np.float32) dictglob['yposstar'] = (np.random.uniform(size=numbstar) * (numbside[1] - 1)).astype(np.float32) dictglob['fluxdistslop'] = fluxdistslop dictglob['minmflux'] = minmflux dictglob['fluxstar'] = minmflux * np.exp(logtflux) dictglob['specstar'] = dictglob['fluxstar'][None, :] dictglob['back'] = np.float32(179.) dictglob['gain'] = np.float32(4.62) # generate galaxies numbgalx = 100 dictglob['numbgalx'] = numbgalx dictglob['xposgalx'] = (np.random.uniform(size=numbgalx) * (numbside[0] - 1)).astype(np.float32) dictglob['yposgalx'] = (np.random.uniform(size=numbgalx) * (numbside[1] - 1)).astype(np.float32) dictglob['sizegalx'] = samp_powr(2., 10., 1.5, size=numbgalx) dictglob['avecfrst'] = (2. * np.random.uniform(size=numbgalx) - 1.).astype( np.float32) dictglob['avecseco'] = (2. * np.random.uniform(size=numbgalx) - 1.).astype( np.float32) dictglob['avecthrd'] = (2. * np.random.uniform(size=numbgalx) - 1.).astype( np.float32) mgtd = np.sqrt(dictglob['avecfrst']**2 + dictglob['avecseco']**2 + dictglob['avecthrd']**2) dictglob['avecfrst'] /= mgtd dictglob['avecseco'] /= mgtd dictglob['avecthrd'] /= mgtd dictglob['specgalx'] = samp_powr(2500., 25000., 2., size=numbgalx)[None, :] dictglob['sbrtgalx'] = retr_sbrt(dictglob['specgalx'], dictglob['sizegalx'], sersindx) gdat = gdatstrt() listsersindx = [0.5, 1., 2., 4., 6., 8., 10.] for sersindx in listsersindx: gdat.sersindx = sersindx gridphon, amplphon = retr_sers(numbsidegrid=numbsidegrid, sersindx=sersindx, pathplot=pathlionimag) gridphon, amplphon = retr_sers(numbsidegrid=numbsidegrid, sersindx=sersindx, pathplot=pathlionimag) numbphongalx = (numbsidegrid + 1)**2 numbener = 1 numbphon = numbgalx * numbphongalx xposphon = np.empty(numbphon) yposphon = np.empty(numbphon) specphon = np.empty((numbener, numbphon)) for k in range(dictglob['numbgalx']): indx = np.arange(k * numbphongalx, (k + 1) * numbphongalx) gridphontemp = gridphon * dictglob['sizegalx'][k] xposphon[indx], yposphon[indx], specphon[:, indx] = retr_tranphon(gridphontemp, amplphon, dictglob['xposgalx'][k], dictglob['yposgalx'][k], dictglob['specgalx'][:, k], \ dictglob['avecfrst'][k], dictglob['avecseco'][k], dictglob['avecthrd'][k], 'fris') xposcand = np.concatenate( (xposphon, dictglob['xposstar'])).astype(np.float32) yposcand = np.concatenate( (yposphon, dictglob['yposstar'])).astype(np.float32) speccand = np.concatenate((specphon, dictglob['specstar']), axis=1).astype(np.float32) indx = np.where((xposcand < 100.) & (xposcand > 0.) & (yposcand < 100.) & (yposcand > 0.))[0] xposcand = xposcand[indx] yposcand = yposcand[indx] speccand = speccand[:, indx] # generate data datacnts = image_model_eval(xposcand, yposcand, speccand[0, :], dictglob['back'], numbside, numbsidepsfn, cpsf) #datacnts[datacnts < 1] = 1. # maybe some negative pixels variance = datacnts / dictglob['gain'] datacnts += (np.sqrt(variance) * np.random.normal(size=(numbside[1], numbside[0]))).astype( np.float32) dictglob['datacnts'] = datacnts # auxiliary data dictglob['numbside'] = numbside dictglob['psfn'] = psfn # write data to disk path = pathliondata + 'true.h5' filearry = h5py.File(path, 'w') for attr, valu in dictglob.iteritems(): filearry.create_dataset(attr, data=valu) filearry.close() gdat.pathlionimag = pathlionimag plot_cntsmaps(gdat, datacnts, 'truedatacnts') plot_cntsmaps(gdat, datacnts, 'truedatacntsphon', xposcand=xposcand, yposcand=yposcand, speccand=speccand)
def clus_popmap(ltfile, maps, map_size, band, name, pixsize, fwhm, loz=None, superplot=0, savemaps=0): # read in the image.dat file ra = [] dec = [] mag = [] with open(ltfile, 'r') as f: data = f.readlines() for i in range(len(data)): val = data[i].strip().split(' ') if i == 0: racent = float(val[-2]) deccent = float(val[-1]) else: ra.append(float(val[1])) dec.append(float(val[2])) mag.append(float(val[-1])) f.close() # if len(loz) == 8 : # ra = [ra,loz['x']] # dec = [dec,loz['y']] refx = float(racent) refy = float(deccent) # convert ra/dec to degrees ra = [((-x / 3600.0) + refx) for x in ra] dec = [((y / 3600.0) + refy) for y in dec] flux = [10.0**(-k / 2.5) for k in mag] # if len(loz) == 8 : # flux = [flux,loz['f']] if superplot: plt.scatter(ra, dec, s=2, c=flux) plt.colorbar() plt.title('Clus Popmap: Pixels') plt.show() header = maps['shead'] wcs = WCS(header) coords = SkyCoord(ra=ra * u.deg, dec=dec * u.deg) x, y = skycoord_to_pixel(coords, wcs) truthtable = {'x': x, 'y': y, 'flux': flux} if superplot: plt.scatter(x, y, s=2, c=flux) plt.colorbar() plt.title('Clus Popmap: Skycoord') plt.show() # make the output map to have noise added later # xf = np.floor(x) # yf = np.floor(y) # cmap = np.zeros((map_size,map_size),dtype=np.float32) # nx, ny = cmap.shape # np.place(xf, xf > nx-1, nx-1) # np.place(yf, yf > ny-1, ny-1) # for cx, cy, cf in zip(xf, yf, flux): # cmap[int(cy), int(cx)] += cf # Note transpose # # beam = get_gauss_beam(fwhm,pixsize,band,oversamp=5) # sim_map = convolve(cmap, beam, boundary='wrap') ''' PCAT LION METHOD''' orig_length = len(x) position_mask = np.logical_and(np.logical_and(x > 0, x < map_size), np.logical_and(y > 0, y < map_size)) x = np.array(x, dtype=np.float32)[position_mask] y = np.array(y, dtype=np.float32)[position_mask] flux = np.array(flux, dtype=np.float32)[position_mask] psf, cf, nc, nbin = get_gaussian_psf_template( fwhm, pixel_fwhm=3.) # assumes pixel fwhm is 3 pixels in each band sim_map = image_model_eval(x, y, nc * flux, 0.0, (int(map_size), int(map_size)), int(nc), cf) '''#############################################################################################################''' if superplot: plt.imshow(sim_map, extent=(0, 300, 0, 300), clim=[0.0, 0.15], origin=0) plt.colorbar() plt.title('Clus Popmap: Lensed map') plt.show() if savemaps: hdx = fits.PrimaryHDU(maps['signal'], maps['shead']) sz = fits.PrimaryHDU(sim_map, hdx.header) if os.path.isfile(config.SIMBOX + 'lensedmap_' + name + '_' + band + '.fits'): os.remove(config.SIMBOX + 'lensedmap_' + name + '_' + band + '.fits') sz.writeto(config.SIMBOX + 'lensedmap_' + name + '_' + band + '.fits') return sim_map, truthtable