def get_usefulcolumns(columns): """ It extracts the vars,evars,posref,zpe,zpo information from a columns file. vars & evars: mag & emags positions inside the catalog. ====USAGE===================================== columns = 'Abell383.columns' vars,evars,posref,zpe,zpo = get_usefulcolumns(columns) ---- """ filt = U.get_str(columns,0) nf = 0 for ii in range(len(filt)): if filt[ii][-4:] == '.res': nf += 1 if filt[ii] == 'M_0': posM0 = ii print 'Number of filters detected... ', nf filtref = int(U.get_str(columns,1)[posM0])-1 rawvars = U.get_str(columns,1,nf) vars = U.zeros(nf) evars = U.zeros(nf) for jj in range(nf): vars[jj] = int(rawvars[jj].split(',')[0])-1 # -1 because of columns's notation evars[jj] = int(rawvars[jj].split(',')[1])-1 # -1 because of columns's notation if vars[jj] == filtref: posref = int(vars[jj]) zpe,zpo = U.get_data(columns,(3,4),nf) vars = vars.astype(int) evars = evars.astype(int) return vars,evars,posref,zpe,zpo
def replacing_nans_catalogs(catalog, newname): """ vars = [] evars = [] data = C.loaddata(catalog) mags = data[:,vars] emags = data[:,evars] """ vars = N.array([ 15, 18, 21, 24, 27, 30, 33, 36, 39, 42, 45, 48, 51, 54, 57, 60, 63, 66, 69, 72, 75, 78, 81, 84, 87, 90, 93, 96, 99, 102, 105, 108, 111 ]) evars = vars[:] + 1 s2n_vars = vars[:] + 2 data = C.loaddata(catalog) # Loading the whole catalog content. head = C.loadheader(catalog) # Loading the original header. mm = data[:, vars] em = data[:, evars] s2n = data[:, s2n_vars] nl = len(mm[:, 0]) # nl is the number of detections inside every single band. nf = len(mm[0, :]) # nf is the number of bands inside the catalog. newmag = U.zeros((nl, nf), float) # Where the new photo errors will be saved. errmag = U.zeros((nl, nf), float) # Where the new photo errors will be saved. new_s2n = U.zeros((nl, nf), float) for jj in range(len(vars)): for ii in range(nl): if abs(mm[ii, jj]) > 60.: newmag[ii, jj] = -99.0 errmag[ii, jj] = 0.00 new_s2n[ii, jj] = -1 elif s2n[ii, jj] < 0.00001: new_s2n[ii, jj] = 0. else: newmag[ii, jj] = mm[ii, jj] errmag[ii, jj] = em[ii, jj] new_s2n[ii, jj] = s2n[ii, jj] # New values of mags error overwrites now the original data. data[:, vars] = newmag[:, U.arange(nf)] data[:, evars] = errmag[:, U.arange(nf)] data[:, s2n_vars] = new_s2n[:, U.arange(nf)] C.savedata(data, newname, dir="", header=head) # Saving & creating a new catalog.
def replace_photo_uncert(catalog, columns): """ """ data = C.loaddata(catalog) # Loading the whole catalog content. head = C.loadheader(catalog) # Loading the original header. mm = A.get_magnitudes(catalog, columns) em = A.get_errmagnitudes(catalog, columns) filters = B.get_filter_list(columns) nl = len(mm[:, 0]) # nl is the number of detections inside every single band. nf = len(mm[0, :]) # nf is the number of bands inside the catalog. errmag = U.zeros((nl, nf), float) # Where the new photo errors will be saved. for jj in range(nf): maglim = B.get_limitingmagnitude(mm[:, jj], em[:, jj], 1., 0.25) print 'Limiting Magnitude for filter %s: %.3f' % (filters[jj], maglim) for ii in range(nl): if mm[ii, jj] == -99.: errmag[ii, jj] = 0.00 elif mm[ii, jj] == 99.: errmag[ii, jj] = maglim else: errmag[ii, jj] = em[ii, jj] # New values of mags error overwrites now the original data. vars, evars, posref, zpe, zpo = get_usefulcolumns(columns) data[:, evars] = errmag[:, U.arange(nf)] finalcatalog = catalog[:-3] + 'upp.cat' C.savedata(data, finalcatalog, dir="", header=head) # Saving & creating a new catalog.
def replace_kerttu_errmags(catalog, columns, finalcatalog): """ import alhambra_kerttu_fixerrmags as AFM catalog = '/Users/albertomolino/doctorado/articulos/ALHAMBRA/kerttu/test_photoz/kerttu.cat' columns = '/Users/albertomolino/doctorado/articulos/ALHAMBRA/kerttu/test_photoz/kerttu.columns' finalcatalog = '/Users/albertomolino/doctorado/articulos/ALHAMBRA/kerttu/test_photoz/kerttu3.cat' AFM.replace_kerttu_errmag(catalog,columns,finalcatalog) ------ """ data = C.loaddata(catalog) # Loading the whole catalog content. head = C.loadheader(catalog) # Loading the original header. mm = A.get_magnitudes(catalog, columns) em = A.get_errmagnitudes(catalog, columns) filters = B.get_filter_list(columns) nl = len(mm[:, 0]) # nl is the number of detections inside every single band. nf = len(mm[0, :]) # nf is the number of bands inside the catalog. errmag = U.zeros((nl, nf), float) # Where the new photo errors will be saved. for jj in range(nf): for ii in range(nl): if mm[ii, jj] == -99.: errmag[ii, jj] = 0.00 else: errmag[ii, jj] = em[ii, jj] # New values of mags error overwrites now the original data. vars, evars, posref, zpe, zpo = A.get_usefulcolumns(columns) data[:, evars] = errmag[:, U.arange(nf)] C.savedata(data, finalcatalog, dir="", header=head) # Saving & creating a new catalog.
def replace_upplimits(cluster): """ """ data = C.loaddata(catalog) # Loading the whole catalog content. head = C.loadheader(catalog) # Loading the original header. m = get_magnitudes(catalog,columns) em = get_errmagnitudes(catalog,columns) filters = bpt.get_filter_list(columns) nl = len(m[:,0]) # nl is the number of detections inside every single band. nf = len(m[0,:]) # nf is the number of bands inside the catalog. errmag = U.zeros((nl,nf),float) # Where the new photo errors will be saved. for jj in range(nf): # maglim = get_limitingmagnitude(m[:,jj],em[:,jj],3.,0.25) maglim = bpt.get_limitingmagnitude(m[:,jj],em[:,jj],1.,0.25) print 'Limiting Magnitude for filter %s: %.3f'%(filters[jj],maglim) for ii in range(nl): # print 'm[%i,%i]'%(ii,jj),m[ii,jj] if m[ii,jj] != 99. : errmag[ii,jj] = em[ii,jj] else: # print 'UNDETECTED OBJECT. SAVING ITS LIMITING MAGNITUDE !!' errmag[ii,jj] = maglim # New values of mags error overwrites now the original data. vars,evars,posref,zpe,zpo = get_usefulcolumns(columns) data[:,evars] = errmag[:,U.arange(nf)] C.savedata(data,finalcatalog, dir="",header=head) # Saving & creating a new catalog.
def inverse_flag_images(image): """ It creates a RMS-like image using its Weight-map where it is defined as RMS = 1 / sqrt(WeightMap) ---- import alhambra_invflagimas as AIV image = '/Volumes/amb22/imagenes/f04/f04p01_F814W_1.swp.weight.flag.fits' AIV.inverse_flag_images(image) """ print 'Processing image: ', image wim = image data = P.open(wim)[0].data nc = len(data[0, :]) nf = len(data[:, 0]) matrix = U.zeros((nf, nc), float) for ii in range(nc): for jj in range(nf): if data[jj, ii] < 1: matrix[jj, ii] = 1 else: matrix[jj, ii] = 0 nameout = A.decapfile(wim) + '.inv.fits' print 'Saving new image as...', nameout P.writeto(nameout, matrix) try: A.addheader2another(image, nameout) except: print 'Impossible to update its header!!!'
def stamping(imagein, coordx, coordy, size): image = imagein xx = coordx yy = coordy data = pyfits.open(image)[0].data matrix = U.zeros((size, size), float) range = size / 2.0 stamp = data[yy - range:yy + range, xx - range:xx + range] return stamp
def flagging_dobledetections_mergecolumns(catalog): """ This serves to append an extra column (each to both inputted catalogs) indicating either a detection was repeated and with the lowest S/N of the two. Sources flagged as 1 are those detections to be excluded when combining both catalogs into a single one. -------- import alhambra_overlap as alhov cat2 = '/Volumes/amb22/catalogos/reduction_v4e/f02/f02p01_colorproext_2_ISO.cat' alhov.flagging_dobledetections_mergecolumns(cat2) """ data = coeio.loaddata(catalog) # Loading the whole catalog content. head = coeio.loadheader(catalog) # Loading the original header. nc = len(data.T) # Number columns dim = len(data[:,0]) # Number elements print 'nc,dim',nc,dim var1 = head[-3].split()[-1] var2 = head[-2].split()[-1] if var1 == var2: print 'Duplicated columns. Merging information...' uno = data[:,72] dos = data[:,73] tres = uno+dos newdata = U.zeros((dim,nc-1),float) for ii in range(nc-1): for jj in range(dim): if ii == nc-1: print 'pepe' newdata[jj,ii] = tres[jj] else: newdata[jj,ii] = data[jj,ii] head2 = head[:-1] head2[-1]='#' outcat = catalog[:-4]+'.mergedcolumns.cat' coeio.savedata(newdata,outcat, dir="",header=head2) # Saving and creating the new catalog. # Renaming files ap.renamefile(catalog,catalog+'.oldold.cat') if not os.path.exists(catalog): ap.renamefile(outcat,catalog)
def remove_detections_bysegmmaps(field, pointing, ccd): """ It uses the segmentation-maps to remove fake detections when masking out saturated stars. ---- import alhambra_fakedets as AF AF.remove_detections_bysegmmaps(2,1,1) """ root = '/Volumes/amb22/catalogos/reduction_v4f/f0%i/' % (field) root2images = '/Volumes/amb22/imagenes/f0%i/' % (field) catalog = root + 'f0%ip0%i_colorproext_%i_ISO.irmsF814W.free.cat' % ( field, pointing, ccd) ids, x, y, area = U.get_data(catalog, (0, 3, 4, 5)) dim = len(ids) valor = U.zeros(dim) ima1 = root2images + 'f0%ip0%i_F814W_%i.swp.seg.fits' % (field, pointing, ccd) ima2 = root2images + 'f0%ip0%i_F814W_%i.swp.segnomask.fits' % ( field, pointing, ccd) segm1 = pyfits.open(ima1)[0].data segm2 = pyfits.open(ima2)[0].data for ii in range(dim): xo = x[ii] yo = y[ii] dimx = U.shape(datos[yo - size:yo + size, xo - size:xo + size])[1] dimy = U.shape(datos[yo - size:yo + size, xo - size:xo + size])[0] perc[ii] = (datos[yo - size:yo + size, xo - size:xo + size].sum() / (dimx * dimy * 1.)) # Defining the sample to be keep. good = U.greater(valor, 0) idr = U.compress(good, ids) dim2 = len(idr) print 'Dimensions: Original: %i, Final: %i, Excluded: %i detections. ' % ( dim, dim2, dim - dim2) finalcat = root + 'f0%ip0%i_colorproext_%i_ISO.irmsF814W.free.cat' % ( field, pointing, ccd) data1 = coeio.loaddata(catalog) # Loading the whole catalog content. head = coeio.loadheader(catalog) data2 = data1[good, :] coeio.savedata(data2, finalcat, dir="", header=head) # Saving & creating a new catalog.
def comparing_populations_HDF5(hdf5file): p = h5py.File(hdf5file, mode='r') pdz = p.get('FullProbability') # pdz = p.get('Likelihood') zz = p.get('redshift')[:] tt = p.get('type')[:] nz = len(zz) ng = N.shape(pdz)[0] probs = U.zeros((nz, 2), float) for ii in range(ng): pepe1 = U.sum(pdzr[ii, :, 0:35], axis=1) pepe2 = U.sum(pdzr[ii, :, 36:], axis=1) pepe3 = (U.sum(pepe1) + U.sum(pepe2)) * 1. # Normalized PDZs pdz1 = pepe1 / pepe3 pdz2 = pepe2 / pepe3 probs[:, 0] += pdz1 probs[:, 1] += pdz2 plt.plot(zz[::30], probs[::30, 0] * 275., 'r-', zz[::30], probs[::30, 1] * 275, 'b-', lw=5) plt.xlim(0., 1.5) for ii in range(ng): pepe1 = U.sum(paca[ii, :, 0:35], axis=1) pepe2 = U.sum(paca[ii, :, 36:], axis=1) pepe3 = (U.sum(pepe1) + U.sum(pepe2)) * 1. # Normalized PDZs pdz1 = pepe1 / pepe3 pdz2 = pepe2 / pepe3 probs[:, 0] += pdz1 probs[:, 1] += pdz2
def remove_fakeabsorptions_F814W(field, pointing, ccd): """ Using the rmsweight images, it gets rid of detections with imrs_F814W < 0.5. ------------------------------------------- import alhambra_fakedets as AF AF.remove_fakeabsorptions_F814W(2,1,1) """ root = '/Volumes/amb22/catalogos/reduction_v4f/f0%i/' % (field) catalog = root + 'f0%ip0%i_colorproext_%i_ISO.cat' % (field, pointing, ccd) ids, x, y, area = U.get_data(catalog, (0, 3, 4, 5)) dim = len(ids) perc = U.zeros(dim) # Opening F814W Weight image ima = alh.alhambra_invrmsimagelist(field, pointing, ccd)[-1] datos = pyfits.open(ima)[0].data for ii in range(dim): if area[ii] > 1: size = int(round(U.sqrt(area[ii]) / 2.)) xo = x[ii] yo = y[ii] dimx = U.shape(datos[yo - size:yo + size, xo - size:xo + size])[1] dimy = U.shape(datos[yo - size:yo + size, xo - size:xo + size])[0] perc[ii] = (datos[yo - size:yo + size, xo - size:xo + size].sum() / (dimx * dimy * 1.)) # Defining the sample to be keep. good = U.greater(perc, 0.5) idr = U.compress(good, ids) dim2 = len(idr) print 'Dimensions: Original: %i, Final: %i, Excluded: %i detections. ' % ( dim, dim2, dim - dim2) finalcat = root + 'f0%ip0%i_colorproext_%i_ISO.irmsF814W.free.cat' % ( field, pointing, ccd) data1 = coeio.loaddata(catalog) # Loading the whole catalog content. head = coeio.loadheader(catalog) data2 = data1[good, :] coeio.savedata(data2, finalcat, dir="", header=head) # Saving & creating a new catalog.
def run_Mastercatalogue(framelist, finalname): """ It generates the MASTER.FILTER HDF5-catalogue with all the individual filters + reference Filter image, according to the list yielded by 'runListing_cats' ------ list_calib_frames,master_filter_cat """ # Make sure the filelist do exist. if not os.path.exists(framelist): print 'The file %s does not exists!' % (framelist) pausa = raw_input('Stop at run_Mastercatalogue!') # /Volumes/amb4/ALHAMBRA/images/individuals/globalist/f02p01c01.info.sort.txt # 1.Filter 2.Date 3.Time 4.EXPT 5.NAME 6.INDEX 7.Pmin 8.Phours 9.Pdays 10.Pyears filt, date, expt, perm, perh, perd, pery = U.get_data( framelist, (0, 1, 3, 6, 7, 8, 9)) # Name of individual align+bias+ZPcal frames frame_names = U.get_str(framelist, 4) # Total number of frames to store. nframes = len(frame_names) # Initializing the HDF5-file. filtros = tb.Filters(complevel=5, complib="lzo") #lz0 is much faster than zlib fp_file = tb.openFile(finalname, mode="w", title="ALHAMBRA VARIAB MASTER CATALOGUE") # Defining and Extracting gral.info from F814W-detection image. ff = framelist.split('/')[-1][2] po = framelist.split('/')[-1][5] ccd = framelist.split('/')[-1][8] cat_f814image = root2cats + 'f0%s/' % ( ff) + 'f0%sp0%s_F814W_%s.swp.cat' % (ff, po, ccd) if not os.path.exists(cat_f814image): print 'File %s does not exists! ' % (cat_f814image) pausa = raw_input('paused inside run_Mastercatalogue.') else: ra, dec, xx, yy, area, fwhm, aa, bb, flag = U.get_data( cat_f814image, (1, 2, 3, 4, 5, 6, 7, 8, 9)) # Retrieving the original ALHAMBRA-ID for each detection. alhids = find_alhambraids(ra, dec) # Estimating the dimensions for the HDF5-file. ng = len(ra) # number of galaxies. nf = nframes # number of frames per galaxy. nc = 15 # number of variables to be stored (see below). # Including this information in the HDF5-file. ids_f814 = fp_file.createArray(fp_file.root, "ALHIDs", alhids) ra_f814 = fp_file.createArray(fp_file.root, "RA", ra) dec_f814 = fp_file.createArray(fp_file.root, "Dec", dec) x_f814 = fp_file.createArray(fp_file.root, "Xpos", xx) y_f814 = fp_file.createArray(fp_file.root, "Ypos", yy) area_f814 = fp_file.createArray(fp_file.root, "Area", area) fwhm_f814 = fp_file.createArray(fp_file.root, "FWHM", fwhm) a_f814 = fp_file.createArray(fp_file.root, "MajorAxis", aa) b_f814 = fp_file.createArray(fp_file.root, "MinorAxis", bb) Flag_f814 = fp_file.createArray(fp_file.root, "SExFlag", flag) # Defining dimension for the main matrix. full_table = fp_file.createCArray(fp_file.root, "AllData", tb.Float32Atom(), shape=(ng, nf, nc), chunkshape=(1, nf, nc), filters=filtros) # Start filling-in the matrix for sss in range(nframes): temporal_frame = frame_names[sss] # print 'Frame to be read: ',temporal_frame # f02p01_954_1.indiv.1.expt500s.20050928.bias.fits ff = temporal_frame.split('.')[0][2] # Extracting the ALHAMBRA-Field frame = temporal_frame.split('.')[2] # Extracting the Temporal-Frame temporal_frame_cat = root2cats + 'f0%s/' % ( ff) + temporal_frame[:-4] + 'ZPcal.cat' # print 'temporal_frame_cat: ',temporal_frame_cat # f02p01_954_1.indiv.1.expt500s.20050928.bias.ZPcal.cat ima_filter_ref = root2images + 'f0%s/' % (ff) + temporal_frame.split( '.')[0] + '.swp.fits' # f02p01_954_1.swp.fits # print 'Assoc. Filter Image: ',ima_filter_ref cat_filter_ref = root2cats + 'f0%s/' % (ff) + A.getfilename( ima_filter_ref)[:-4] + 'cat' # f02p01_954_1.swp.cat # print 'Assoc. Filter Catalogue: ',cat_filter_ref # Reading information from Catalogues: # from Frame catalogue: fl, efl, m, em = U.get_data(temporal_frame_cat, (10, 11, 12, 13)) S2N = (fl / efl) # from Reference Filter Catalogue. if sss < 1: current_cat = cat_filter_ref flr, eflr, mr, emr = U.get_data(cat_filter_ref, (10, 11, 12, 13)) # This prevents it to re-read the same reference catalogue # for indiv.frames with the same filter. if sss > 0 and cat_filter_ref != current_cat: flr, eflr, mr, emr = U.get_data(cat_filter_ref, (10, 11, 12, 13)) S2Nr = (flr / eflr) else: S2Nr = (flr / eflr) # Writing information into the HDF5-File. # Frame > Galaxies > Columns for iii in range(ng): values = U.zeros(nc) # 1.Frame 2.Mag 3.dMag 4.Flux 5.dFlux 6.MagRef 7.dMagRef # 8. FluxRef 9.dFluxRef 10.S2N 11.S2NRef 12.ExpTime # 13. TimeLapse['] 14.Date 15.Filter values[0] = frame # 1. Frame Number (1,2,3,...). values[1] = m[iii] # 2. ISO.Magnitude on indiv.frame. values[2] = em[iii] # 3. Unc.ISO.Magnitude on indiv.frame. values[3] = fl[iii] # 4. ISO.Flux on indiv.frame. values[4] = efl[iii] # 5. Unc.ISO.Flux on indiv.frame. values[5] = mr[iii] # 6. ISO.Magnitude on Ref.Filter. values[6] = emr[iii] # 7. Unc.ISO.Magnitude on Ref.Filter. values[7] = flr[iii] # 8. ISO.Flux on Ref.Filter. values[8] = eflr[iii] # 9. Unc.ISO.Flux on Ref.Filter. values[9] = S2N[iii] # 10. signal-to-noise on indiv.frame. values[10] = S2Nr[iii] # 11. signal-to-noise on Ref.Filter. values[11] = expt[sss] # 12. Exposure-Time on indiv.frame. values[12] = perm[ sss] # 13. Time-Lapse from First Observation [sec] values[13] = date[sss] # 14. Date when indiv.frame was observed. values[14] = filt[sss] # 15. Filter for indiv.frame full_table[iii, sss, :] = values[:] fp_file.close() master_cat = root2cats + 'f0%s/f0%sp0%sc0%s.hdf5' % (ff, ff, po, ccd) if not os.path.exists(master_cat): cmd5 = '/bin/mv %s %s ' % (finalname, master_cat) print cmd5 os.system(cmd5) if os.path.exists(finalname): print '=============================================================' print 'The HDF5-file %s was successfully created.' % ( A.getfilename(finalname)) print '=============================================================' cmd6 = '/bin/rm -rf %s ' % (finalname) # print cmd6 os.system(cmd6)
def match_spz_sample(cluster): # TO CHECK finalcat1 = catalog2[:-3]+'CLASH.redu.cat' finalcat2 = catalog2[:-3]+'nada.cat' # if not os.path.exists(finalcat1): if not os.path.exists(finalcat2): # print 'Final catalog does not exist yet.' if os.path.exists(catalog1) and os.path.exists(catalog2): # It matches up detections to its Spectroscopic Sample. # Reading specz catalog print 'Reading info1 before matching...' speczsample = catalog1 idsp,xsp,ysp = U.get_data(speczsample,(0,3,4)) goodsp = U.greater_equal(xsp,1500) * U.less_equal(xsp,3500) goodsp *= U.greater_equal(ysp,1500) * U.less_equal(ysp,3500) idsp,xsp,ysp = U.multicompress(goodsp,(idsp,xsp,ysp)) print 'New dimension for specz catalogue: ',len(xsp) # rasp,decsp,xsp,ysp,zsp = get_data(speczsample,(0,1,2,3,4)) # xsp,ysp,zsp = get_data(speczsample,(1,2,7)) ####### idsp = U.arange(len(xsp))+1 # idsp = arange(len(rasp))+1 # Reading ColorPro catalog print 'Reading info2 before matching...' idcol,xcol,ycol = U.get_data(catalog2,(0,3,4)) print 'Dimension for input catalogue before compressing: ',len(idcol) gsp = U.greater_equal(xcol,1500) * U.less_equal(xcol,3500) gsp *= U.greater_equal(ycol,1500) * U.less_equal(ycol,3500) idcol,xcol,ycol = U.multicompress(gsp,(idcol,xcol,ycol)) print 'Dimension for input catalogue after compressing: ',len(idcol) # Using "matching_vects" to match up samples... print 'Matching samples....' pepe = CT.matching_vects(idcol,xcol,ycol,idsp,xsp,ysp,1.1) # We use now X,Y instead RA,Dec # Compressing matches for ColorPro... print 'Compressing matches...' matchidcol = pepe[:,0].astype(int) gdet_col = U.greater(matchidcol,0) # Excluding 0's (non matched detections) matchidcol = U.compress(gdet_col,(matchidcol)) # Compressing matches for Spectroscopic... matchidsp = pepe[:,1].astype(int) gdet_spz = U.greater(matchidsp,0) # Excluding 0's (non matched detections) matchidsp = U.compress(gdet_spz,(matchidsp)) print 'len(idcol)',len(idcol) print 'len(idsp)',len(idsp) if len(matchidcol) == len(matchidsp): print 'Creating idredu & zsredu ' print 'Dimension of matchidsp ',len(matchidsp) idredu = U.zeros(len(matchidsp)) idspredu = U.zeros(len(matchidsp)) for ii in range(len(matchidsp)): colindex = A.id2pos(idcol,matchidcol[ii]) # Position for Index idcol spzindex = A.id2pos(idsp,matchidsp[ii]) # Position for Index idsp idredu[ii] = idcol[colindex] # ID for ColorPro idspredu[ii] = idsp[spzindex] # Specz for Specz # A new smaller catalog will be created containing specz info as an extra column. print 'Selecting by rows... ' finalcat1 = catalog2[:-3]+'UDF.redu.cat' finalcat2 = catalog2[:-3]+'CLASH.redu.cat' U.put_data(catalog2[:-3]+'idsfrommatch.txt',(idredu,idspredu)) A.select_rows_bylist_sorted(catalog1,idspredu,finalcat1) A.select_rows_bylist_sorted(catalog2,idredu,finalcat2)
def appending_ids2catalogues(field, pointing, ccd): """ import alhambra_3arcs as A3 A3.appending_ids2catalogues(2,1,1) """ catalhambra = root + 'f0%i/alhambra.F0%iP0%iC0%i.ColorProBPZ.cat' % ( field, field, pointing, ccd) idalh = U.get_str(catalhambra, 0) idalh2 = U.arange(len(idalh)) + 1 xalh, yalh = U.get_data(catalhambra, (6, 7)) cat3arcs = finalroot + 'f0%i/alhambra.f0%ip0%ic0%i.3arcs.cat' % ( field, field, pointing, ccd) id3arcs, x3arcs, y3arcs = U.get_data(cat3arcs, (0, 3, 4)) print len(id3arcs) matchfile = cat3arcs[:-3] + 'idsfrommatch.txt' if not os.path.exists(matchfile): idcol = idalh2 xcol = xalh ycol = yalh idsp = id3arcs xsp = x3arcs ysp = y3arcs pepe = CT.matching_vects(idcol, xcol, ycol, idsp, xsp, ysp, 5) # Compressing matches for ColorPro... print 'Compressing matches...' matchidcol = pepe[:, 0].astype(int) gdet_col = U.greater(matchidcol, 0) # Excluding 0's (non matched detections) matchidcol = U.compress(gdet_col, (matchidcol)) # Compressing matches for Spectroscopic... matchidsp = pepe[:, 1].astype(int) gdet_spz = U.greater(matchidsp, 0) # Excluding 0's (non matched detections) matchidsp = U.compress(gdet_spz, (matchidsp)) print 'len(idcol)', len(idcol) print 'len(idsp)', len(idsp) if len(matchidcol) == len(matchidsp): print 'Creating idredu & zsredu ' print 'Dimension of matchidsp ', len(matchidsp) idredu = U.zeros(len(matchidsp)) idspredu = U.zeros(len(matchidsp)) for ii in range(len(matchidsp)): colindex = A.id2pos(idcol, matchidcol[ii]) # Position for Index idcol spzindex = A.id2pos(idsp, matchidsp[ii]) # Position for Index idsp idredu[ii] = idcol[colindex] # ID for ColorPro idspredu[ii] = idsp[spzindex] # Specz for Specz matchfile = cat3arcs[:-3] + 'idsfrommatch.txt' U.put_data(matchfile, (idredu, idspredu)) if os.path.exists(matchfile): pepa = open(matchfile[:-3] + 'bis.cat', 'w') idredu, idspredu = U.get_data(matchfile, (0, 1)) i11 = idredu.astype(int) - 1 i22 = idspredu.astype(int) lista = [] for ii in range(len(i11)): lista.append(idalh[i11[ii]]) pepa.write('%s %s \n' % (idalh[i11[ii]], i22[ii])) pepa.close() finalfinal = cat3arcs[:-3] + 'final.cat' if os.path.exists(finalfinal): A.deletefile(finalfinal) if not os.path.exists(finalfinal): print 'Preparing ', finalfinal idsa = U.get_str(matchfile[:-3] + 'bis.cat', 0) append_IDs2_3arcs_catalogues(cat3arcs, idsa)
def flagging_dobledetections(cat1,cat2): """ This serves to append an extra column (each to both inputted catalogs) indicating either a detection was repeated and with the lowest S/N of the two. Sources flagged as 1 are those detections to be excluded when combining both catalogs into a single one. -------- import alhambra_overlap as alhov cat1 = '/Volumes/amb22/catalogos/reduction_v4e/f02/f02p02_colorproext_1_ISO.cat' cat2 = '/Volumes/amb22/catalogos/reduction_v4e/f02/f02p01_colorproext_1_ISO.cat' alhov.flagging_dobledetections(cat1,cat2) """ id1,ra1,dec1,x1,y1,s2n1 = U.get_data(cat1,(0,1,2,3,4,14)) id2,ra2,dec2,x2,y2,s2n2 = U.get_data(cat2,(0,1,2,3,4,14)) ne1 = len(id1) ne2 = len(id2) g1 = U.greater_equal(ra1,min(ra2)) g2 = U.less_equal(ra2,max(ra1)) id1r,ra1r,dec1r,x1r,y1r,s2n1r = U.multicompress(g1,(id1,ra1,dec1,x1,y1,s2n1)) id2r,ra2r,dec2r,x2r,y2r,s2n2r = U.multicompress(g2,(id2,ra2,dec2,x2,y2,s2n2)) flag1 = U.zeros(ne1) flag2 = U.zeros(ne2) dim1 = len(id1r) dim2 = len(id2r) print 'dim1,dim2',dim1,dim2 if dim1>0 and dim2>0: print 'Matching samples....' pepe = matching_vects_ddet(id1r,ra1r,dec1r,id2r,ra2r,dec2r,0.000312) # We use now X,Y instead RA,Dec # Purging null elements matchidcol = pepe[:,0].astype(int) good_det1 = U.greater(matchidcol,0) # Excluding 0's (non matched detections) matchidcol = U.compress(good_det1,(matchidcol)) matchidsp = pepe[:,1].astype(int) good_det2 = U.greater(matchidsp,0) # Excluding 0's (non matched detections) matchidsp = U.compress(good_det2,(matchidsp)) if len(matchidcol) == len(matchidsp) and len(matchidcol) >0 : newdim = len(matchidsp) print 'Dimension of matching',newdim idr1 = U.zeros(newdim) idr2 = U.zeros(newdim) s2nr1 = U.zeros(newdim) s2nr2 = U.zeros(newdim) for ii in range(newdim): idr1index = ap.id2pos(id1r,matchidcol[ii]) idr2index = ap.id2pos(id2r,matchidsp[ii]) idr1[ii] = id1r[idr1index] s2nr1[ii] = s2n1r[idr1index] idr2[ii] = id2r[idr2index] s2nr2[ii] = s2n2r[idr2index] # Select/Purge detections according to its S/N marcador1 = U.zeros(newdim) marcador2 = U.zeros(newdim) for ss in range(newdim): cociente = s2nr1[ss]/s2nr2[ss] if cociente >= 1.: marcador1[ss] = 1. else: marcador2[ss] = 1. cond1 = U.less(marcador1,1) cond2 = U.less(marcador2,1) idr1b = U.compress(cond1,idr1) dim1rr = len(idr1b) idr2b = U.compress(cond2,idr2) dim2rr = len(idr2b) # Two new IDs (finalid1 & finalid2) are generated with # the final elements to be included in the output catalog. for hh1 in range(ne1): if id1[hh1] in idr1b: flag1[hh1] = 1 for hh2 in range(ne2): if id2[hh2] in idr2b: flag2[hh2] = 1 # A new smaller catalog will be created containing specz info as an extra column. outcat1 = ap.decapfile(cat1)+'.doubledetect.cat' outcat2 = ap.decapfile(cat2)+'.doubledetect.cat' print 'outcat1',outcat1 print 'outcat2',outcat2 ap.appendcol(cat1,flag1,'Flag2Detected',outcat1) ap.appendcol(cat2,flag2,'Flag2Detected',outcat2) # Renaming files ap.renamefile(cat1,cat1+'.old.cat') if not os.path.exists(cat1): ap.renamefile(outcat1,cat1) ap.renamefile(cat2,cat2+'.old.cat') if not os.path.exists(cat2): ap.renamefile(outcat2,cat2) else: print 'No common sources in betwen the catalogs' # A new smaller catalog will be created containing specz info as an extra column. outcat1 = ap.decapfile(cat1)+'.doubledetect.cat' outcat2 = ap.decapfile(cat2)+'.doubledetect.cat' print 'outcat1',outcat1 print 'outcat2',outcat2 ap.appendcol(cat1,flag1*0,'Flag2Detected',outcat1) ap.appendcol(cat2,flag2*0,'Flag2Detected',outcat2) # Renaming files ap.renamefile(cat1,cat1+'.old.cat') if not os.path.exists(cat1): ap.renamefile(outcat1,cat1) ap.renamefile(cat2,cat2+'.old.cat') if not os.path.exists(cat2): ap.renamefile(outcat2,cat2)
def matching_vects_ddet(c10,c11,c12,c20,c21,c22,precision): """ ------------------------------------------------------------------------- The program matchs objects using their coordinates. ------------------------------------------------------------------------- c10: identification number from set1 c11 & c12: coordinates to be used when matching the common objects. c20: identification number from set2. c21 & c22: coordinates to be used when matching the common objects. An output matrix (outmatrix) will provide the matched elements. outmatrix[0]: c10_matched & outmatrix[1]: c20_matched -------------------------------------------------------------------------- Alberto Molino amb.at.iaa.es // July_09 // -------------------------------------------------------------------------- """ # Variable definition. id1= c10 x1 = c11 y1 = c12 dim1 = len(c10) id2= c20 x2 = c21 y2 = c22 dim2 = len(c20) delta_xx = U.zeros(dim1+dim2,float) delta_yy = U.zeros(dim1+dim2,float) INSIDE = U.zeros((dim1+dim2,2),float) MATCHING = U.zeros((dim1+dim2,2),float) kmin = 0 nn = 0 prec = float(precision) thr = 1e-30 for jj in range(dim1): kk = 0 for ii in range(dim2): delta_xx[ii] = (x2[ii]-x1[jj]) delta_yy[ii] = (y2[ii]-y1[jj]) circle = U.sqrt(delta_xx[ii]**2 + delta_yy[ii]**2) if circle < prec: INSIDE[kk,0]=float(circle) INSIDE[kk,1]=id2[ii] kk += 1 kmin = 0 if kk > 0: if kk > 1: kmin = U.argmin(INSIDE[0:kk,0]) MATCHING[nn,0]=id1[jj] MATCHING[nn,1]=INSIDE[kmin,1] # id2 else: MATCHING[nn,0]=id1[jj] MATCHING[nn,1]=INSIDE[0,1] # id2 nn += 1 # Real dimension of matched objects print '-------------------------------------' print ' Matched elements= ',nn if nn > 0 : print MATCHING[0:nn,0] print MATCHING[0:nn,1] return MATCHING
def purging_dobledetections(cat1,cat2): """ import alhambra_overlap from alhambra_overlap import * cat1 = '/Volumes/amb22/catalogos/reduction_v4e/f02/f02p02_colorproext_1_ISO.cat' cat2 = '/Volumes/amb22/catalogos/reduction_v4e/f02/f02p01_colorproext_1_ISO.cat' purging_dobledetections(cat1,cat2) """ id1,ra1,dec1,x1,y1,s2n1 = U.get_data(cat1,(0,1,2,3,4,14)) id2,ra2,dec2,x2,y2,s2n2 = U.get_data(cat2,(0,1,2,3,4,14)) ne1 = len(id1) ne2 = len(id2) g1 = U.greater_equal(ra1,min(ra2)) g2 = U.less_equal(ra2,max(ra1)) id1r,ra1r,dec1r,x1r,y1r,s2n1r = U.multicompress(g1,(id1,ra1,dec1,x1,y1,s2n1)) id2r,ra2r,dec2r,x2r,y2r,s2n2r = U.multicompress(g2,(id2,ra2,dec2,x2,y2,s2n2)) dim1 = len(id1r) dim2 = len(id2r) print 'dim1,dim2',dim1,dim2 if dim1>0 and dim2>0: print 'Matching samples....' pepe = matching_vects_ddet(id1r,ra1r,dec1r,id2r,ra2r,dec2r,0.000312) # We use now X,Y instead RA,Dec # Purging null elements matchidcol = pepe[:,0].astype(int) good_det1 = U.greater(matchidcol,0) # Excluding 0's (non matched detections) matchidcol = U.compress(good_det1,(matchidcol)) matchidsp = pepe[:,1].astype(int) good_det2 = U.greater(matchidsp,0) # Excluding 0's (non matched detections) matchidsp = U.compress(good_det2,(matchidsp)) if len(matchidcol) == len(matchidsp) and len(matchidcol) >0 : newdim = len(matchidsp) print 'Dimension of matching',newdim idr1 = U.zeros(newdim) idr2 = U.zeros(newdim) s2nr1 = U.zeros(newdim) s2nr2 = U.zeros(newdim) for ii in range(newdim): idr1index = ap.id2pos(id1r,matchidcol[ii]) idr2index = ap.id2pos(id2r,matchidsp[ii]) idr1[ii] = id1r[idr1index] s2nr1[ii] = s2n1r[idr1index] idr2[ii] = id2r[idr2index] s2nr2[ii] = s2n2r[idr2index] # Select/Purge detections according to its S/N marcador1 = U.zeros(newdim) marcador2 = U.zeros(newdim) for ss in range(newdim): cociente = s2nr1[ss]/s2nr2[ss] if cociente >= 1.: marcador1[ss] = 1. else: marcador2[ss] = 1. cond1 = U.less(marcador1,1) cond2 = U.less(marcador2,1) idr1b = U.compress(cond1,idr1) dim1rr = len(idr1b) idr2b = U.compress(cond2,idr2) dim2rr = len(idr2b) print '' print 'Number of detections to be removed from cat1: ', dim1rr print 'Number of detections to be removed from cat2: ', dim2rr print '' # Two new IDs (finalid1 & finalid2) are generated with # the final elements to be included in the output catalog. finalid1 = U.zeros((ne1-dim1rr)) finalid2 = U.zeros((ne2-dim2rr)) kk1 = 0 for hh1 in range(ne1): if id1[hh1] not in idr1b: finalid1[kk1] = id1[hh1] kk1 += 1 print 'kk1',kk1 kk2 = 0 for hh2 in range(ne2): if id2[hh2] not in idr2b: if kk2 <= (ne2-dim2rr-1): finalid2[kk2] = id2[hh2] kk2+=1 print 'kk2',kk2 # A new smaller catalog will be created containing specz info as an extra column. outcat1 = ap.decapfile(cat1)+'.wo2detect.cat' outcat2 = ap.decapfile(cat2)+'.wo2detect.cat' print 'outcat1',outcat1 print 'outcat2',outcat2 ap.select_rows_bylist(cat1,finalid1,outcat1) ap.select_rows_bylist(cat2,finalid2,outcat2) else: print 'No common sources in betwen the catalogs'
nl = len(globalist) for ii in range(nl): # imas = U.get_str(globalist[ii],0) # ni = len(imas) infofile_name = globalist[ii][:-4] + 'info.txt' fi, da, ti, ex, ix = U.get_data(infofile_name, (0, 1, 2, 3, 5)) im, ti2 = U.get_str(infofile_name, (4, 2)) ni = len(im) infofile_name_sorted = globalist[ii][:-4] + 'info.sort.txt' print 'Creating ', infofile_name_sorted infofile = open(infofile_name_sorted, 'w') header = '# FILTER DATE TIME EXPTIME NAME INDEX PERIOD [min] PERIOD [hours] PERIOD [days] PERIOD [years] \n' infofile.write(header) newdate = U.zeros(ni, 'int') for ii in range(ni): uno = int(da[ii]) dos = ti2[ii] ll = '%s%s' % (uno, dos) newdate[ii] = float(ll) newdatasor = U.sort(newdate) # # Sorted by date # fir,dar,tir,exr,imr,ixr,ti2r = U.multisort(da,(fi,da,ti,ex,im,ix,ti2)) # # sorted by time # fir2,dar2,ti2r2,exr2,imr2,ixr2 = U.multisort(tir,(fir,dar,ti2r,exr,imr,ixr)) fir, dar, tir, exr, imr, ixr, ti2r = U.multisort( newdate, (fi, da, ti, ex, im, ix, ti2)) period = U.zeros(ni, 'int')
def run_getframelist(field, pointing, ccd): """ It creates the bias-free image list for a given FPC. """ ff = field po = pointing ccd = ccd refF814ima = root2f814 + 'f0%s/f0%sp0%s_F814W_%s.swp.fits' % (ff, ff, po, ccd) if os.path.exists(refF814ima): listaname = root2lists + 'f0%sp0%sc0%s.list' % (ff, po, ccd) imas = root2alig + 'f0%s/f0%sp0%s_*_%s.indiv.*.fits' % (ff, ff, po, ccd) cmd = 'ls %s > %s' % (imas, listaname) print cmd os.system(cmd) if os.path.exists(listaname): globalist = U.get_str(listaname, 0) ni = len(globalist) infofile_name = listaname[:-4] + 'info.txt' infofile = open(infofile_name, 'w') header = '# FILTER DATE TIME EXPTIME NAME INDEX \n' infofile.write(header) for ss in range(ni): nick = globalist[ss].split('/')[-1] ff = nick[2:3] po = nick[5:6] ccd = nick[11:12] filt = nick[7:10] date = nick.split('.')[4] expt = nick.split('.')[3][4:][:-1] try: head = fits.getheader(globalist[ss]) # head = pyfits.open(globalist[ss])[0].header hours = head['DATE-OBS'].split('T')[1].split(':')[0] minut = head['DATE-OBS'].split('T')[1].split(':')[1] secs = head['DATE-OBS'].split('T')[1].split(':')[2] reloj = hours + minut + secs print 'image: ', nick linea = '%s %s %s %s %s %i \n' % ( filt, date, reloj, expt, nick, ss + 1) infofile.write(linea) except: continue infofile.close() if os.path.exists(infofile_name): fi, da, ti, ex, ix = U.get_data(infofile_name, (0, 1, 2, 3, 5)) im, ti2 = U.get_str(infofile_name, (4, 2)) ni = len(im) infofile_name_sorted = infofile_name[:-4] + '.sort.txt' print 'Creating ', infofile_name_sorted infofile = open(infofile_name_sorted, 'w') header = '# FILTER DATE TIME EXPTIME NAME INDEX PERIOD [min] PERIOD [hours] PERIOD [days] PERIOD [years] \n' infofile.write(header) # Sorting frames by date newdate = U.ones(ni, dtype='int') for ii in range(ni): uno = int(da[ii]) dos = ti2[ii] ll = '%s%s' % (uno, dos) newdate[ii] = float(ll) newdatasor = U.sort(newdate) fir, dar, tir, exr, imr, ixr, ti2r = U.multisort( newdate, (fi, da, ti, ex, im, ix, ti2)) # Estimating Time-Lapses ('period') among observations. period = U.zeros(ni, dtype='Float64') # period = U.ones(ni,dtype='int') for ii in range(ni - 1): period[ii + 1] = A.get_observ_freq(int(dar[0]), ti2r[0], int(dar[ii + 1]), ti2r[ii + 1]) for ss in range(ni): linea = '%i %i %s %i %s %i %i %i %i %i \n' % ( fir[ss], dar[ss], ti2r[ss], exr[ss], imr[ss], ixr[ss], period[ss] / 60., period[ss] / 3600., period[ss] / 86400., period[ss] / 31536000.) infofile.write(linea) infofile.close() print ' '
def singlemosaico(images, coox, cooy, size, nt, nx, ny, imageout): """ ================================================================ It generates a single mosaic-like image according to the inputed image and the coordinates (X,Y) ---------- imagein: image to be used. coox: 1D Integer vector with Axis-X coordinates [pixel] cooy: 1D Integer vector with Axis-Y coordinates [pixel] size: size for individual internal substamps nt: number of total substamps (numb. objects) nx: number of horizontal stamps ny: number of vertical stamps imageout: final name for the mosaic image ================================================================ USAGE: ---------- import mosaicing from mosaicing import * image = '/Volumes/amb2/SUBARU/CLJ1226/images/clj1226_Z_2003_20130227_sw.fits' size = 25 nt = 418 nx=21 ny=20 coox,cooy = U.get_data('/Volumes/amb2/SUBARU/CLJ1226/catalogs/stars.cat',(0,1)) imageout = '/Volumes/amb2/SUBARU/CLJ1226/images/clj1226_Z_2003_20130227_sw.mosaic.fits' singlemosaic(image,coox,cooy,size,nt,nx,ny,imageout) ================================================================= """ # Definition of variables. mad = size # Size of every sub-squared-stamp. 100 is a good choice! mad2 = mad + 1 albumdata = U.zeros((ny * mad2 + 1, nx * mad2 + 1), float) # coox = coox.astype(int) # cooy = cooy.astype(int) print ' --------------------------------------------------------------------------- ' print ' A stamp size = %d x %d has been chosen ' % (mad, mad) print ' One galaxy will be display in a %d x %d album-like image' % (nx, ny) print ' --------------------------------------------------------------------------- ' for i in range(nt): ix = i % nx iy = ny - (i / nx) - 1 # It picks the ith-submatrix from the ith image. # So, It creates every single sub-mosaic! print 'images[i],coox[i],cooy[i],mad' if nt == 1: stamp = alhmos.stamping(images, coox, cooy, mad) else: stamp = alhmos.stamping(images[i], coox[i], cooy[i], mad) ax = ix * mad2 + 1 ay = iy * mad2 + 1 # Saving the ith-submosaic in albumdata. albumdata[ay:ay + mad, ax:ax + mad] = stamp.astype(float) print ' Copying submosaic %i from image %i: ' % (i, nt) # Creating the new mosaic as a fits file. pyfits.writeto(imageout, albumdata, clobber=True)