def main(): ###make sure to change these when running in a new enviorment!### #location of data directory filepath_cat1 = cu.get_output_path() + 'processed_data/yang_groupcat/' filepath_cat2 = cu.get_output_path() + 'processed_data/mpa_dr7/' #save data to directory... savepath1 = filepath_cat1+'mpa_dr7_match/' savepath2 = filepath_cat2+'yang_groupcat_match/' ################################################################# catalogues_1=['sample1_L_petro','sample2_L_petro','sample3_L_petro',\ 'sample1_M_petro','sample2_M_petro','sample3_M_petro',\ 'sample1_L_model','sample2_L_model','sample3_L_model',\ 'sample1_M_model','sample2_M_model','sample3_M_model'] catalogues_2=['gal_info_gal_totspecsfr_dr7_v5_2'] for catalogue in catalogues_1: catalogue1 = catalogue catalogue2 = catalogues_2[0] print catalogue1,'match into', catalogue2 f1 = h5py.File(filepath_cat1+catalogue1+'.hdf5', 'r') #open catalogue file GC = f1.get(catalogue1) f2 = h5py.File(filepath_cat2+catalogue2+'.hdf5', 'r') #open catalogue file W = f2.get(catalogue2) da=2.0*1.0/3600.0 #matching length result_1 = np.array(cu.spherematch(GC['RAgal'], GC['DECgal'], W['RA'], W['DEC'], tol=da, nnearest=1)) #result_2 = np.array(cu.spherematch(GC['RAgal'], GC['DECgal'], W['RA'], W['DEC'], tol=da, nnearest=2)) #not used repeats = [item for item, count in Counter(result_1[1]).iteritems() if count > 1] #double matched objects if len(repeats) > 0: remove=np.zeros((0,),dtype=np.int) #which entries to remove for repeat in repeats: result_a = np.where(result_1[1]==repeat)[0] #find indices of the double matched object into catalogue2 result_b = np.where(result_1[2][result_a]>np.min(result_1[2][result_a]))[0] #find less good matches result_c = result_a[result_b] #indices of less good matches into catalogue2 remove = np.hstack((remove,result_c)) #indices which should be removed keep = np.arange(0,len(result_1[0]),1).astype(int) keep = np.in1d(keep,remove)==False result = result_1[:,keep] #unique = np.setdiff1d(result_2[0], result_1[0]) #not used else: result = result filename1 = catalogue2+'_'+catalogue1+'_match' filename2 = catalogue1+'_'+catalogue2+'_match' np.save(savepath1+filename1, result[1].astype(int)) np.save(savepath2+filename2, result[0].astype(int))
def main(): ###make sure to change these when running in a new enviorment!### #location of data directories filepath_1 = cu.get_output_path() + 'processed_data/mpa_dr7/' filepath_2 = cu.get_output_path() + 'processed_data/NYU_VAGC/' ################################################################# catalogue_1 = 'mpa_dr7_unique' catalogue_2 = 'nyu_vagc_dr7' f_1 = h5py.File(filepath_1+catalogue_1+'.hdf5', 'r') dset_1 = f_1.get(catalogue_1) dset_1 = np.array(dset_1) #print(dset_1.dtype.names) f_2 = h5py.File(filepath_2+catalogue_2+'.hdf5', 'r') dset_2 = f_2.get(catalogue_2) dset_2 = np.array(dset_2) #print(dset_2.dtype.names) print("number of objects in MPA-JHU catalogue: {0}".format(len(dset_1))) print("number of objects in NYU catalogue: {0}".format(len(dset_2))) da = 1/3600.0 * 2.0 #match length is 2" ind1, ind2, ds = cu.spherematch(dset_1['RA'], dset_1['DEC'],\ dset_2['RA'], dset_2['DEC'],\ tol=da, nnearest=1) print("minimum angular seperation: {0}''".format(np.min(ds)*3600.0)) print("maximum angular seperation: {0}''".format(np.max(ds)*3600.0)) print("number of matchs: {0}".format(len(ind1))) print(dset_1[ind1]['Z']) print(dset_2[ind2]['Z']) #save matching file filename = 'matches_into_nyu_vagc' np.save(filepath_1+'matches/'+filename,ind1) filename = 'nyu_vagc_matched_to_mpa' np.save(filepath_1+'matches/'+filename,ind1)
def main(): ###make sure to change these when running in a new enviorment!### #location of data directories filepath_1 = cu.get_output_path() + 'processed_data/mpa_dr7/' filepath_2 = cu.get_output_path() + 'processed_data/NYU_VAGC/' ################################################################# catalogue_1 = 'mpa_dr7_unique' catalogue_2 = 'nyu_vagc_dr7' f_1 = h5py.File(filepath_1 + catalogue_1 + '.hdf5', 'r') dset_1 = f_1.get(catalogue_1) dset_1 = np.array(dset_1) #print(dset_1.dtype.names) f_2 = h5py.File(filepath_2 + catalogue_2 + '.hdf5', 'r') dset_2 = f_2.get(catalogue_2) dset_2 = np.array(dset_2) #print(dset_2.dtype.names) print("number of objects in MPA-JHU catalogue: {0}".format(len(dset_1))) print("number of objects in NYU catalogue: {0}".format(len(dset_2))) da = 1 / 3600.0 * 2.0 #match length is 2" ind1, ind2, ds = cu.spherematch(dset_1['RA'], dset_1['DEC'],\ dset_2['RA'], dset_2['DEC'],\ tol=da, nnearest=1) print("minimum angular seperation: {0}''".format(np.min(ds) * 3600.0)) print("maximum angular seperation: {0}''".format(np.max(ds) * 3600.0)) print("number of matchs: {0}".format(len(ind1))) print(dset_1[ind1]['Z']) print(dset_2[ind2]['Z']) #save matching file filename = 'matches_into_nyu_vagc' np.save(filepath_1 + 'matches/' + filename, ind1) filename = 'nyu_vagc_matched_to_mpa' np.save(filepath_1 + 'matches/' + filename, ind1)
def main(): ###make sure to change these when running in a new enviorment!### #location of data directory filepath_cat1 = cu.get_output_path() + 'processed_data/yang_groupcat/' filepath_cat2 = cu.get_output_path() + 'processed_data/NYU_VAGC/' #save data to directory... savepath1 = filepath_cat1+'nyu_vagc_match/' savepath2 = filepath_cat2+'yang_groupcat_match/' ################################################################# catalogues_1 = ['sample1_L_petro','sample2_L_petro','sample3_L_petro',\ 'sample1_M_petro','sample2_M_petro','sample3_M_petro',\ 'sample1_L_model','sample2_L_model','sample3_L_model',\ 'sample1_M_model','sample2_M_model','sample3_M_model'] catalogues_2 = ['nyu_vagc_dr7'] for catalogue in catalogues_1: catalogue1 = catalogue catalogue2 = catalogues_2[0] print catalogue1,'match into', catalogue2 f1 = h5py.File(filepath_cat1+catalogue1+'.hdf5', 'r') #open catalogue file GC = f1.get(catalogue1) f2 = h5py.File(filepath_cat2+catalogue2+'.hdf5', 'r') #open catalogue file W = f2.get(catalogue2) da=2.0*1.0/3600.0 #matching length in degrees result_1 = np.array(cu.spherematch(GC['RAgal'], GC['DECgal'], W['RA'], W['DEC'], tol=da, nnearest=1)) #result_2 = np.array(cu.spherematch(GC['RAgal'], GC['DECgal'], W['RA'], W['DEC'], tol=da, nnearest=2)) #not used repeats = [item for item, count in Counter(result_1[1]).iteritems() if count > 1] #double matched objects if len(repeats) > 0: remove=np.zeros((0,),dtype=np.int) #which entries to remove for repeat in repeats: result_a = np.where(result_1[1]==repeat)[0] #find indices of the double matched object into catalogue2 result_b = np.where(result_1[2][result_a]>np.min(result_1[2][result_a]))[0] #find less good matches result_c = result_a[result_b] #indices of less good matches into catalogue2 remove = np.hstack((remove,result_c)) #indices which should be removed keep = np.arange(0,len(result_1[0]),1).astype(int) keep = np.in1d(keep,remove)==False result = result_1[:,keep] #unique = np.setdiff1d(result_2[0], result_1[0]) #not used else: result = result_1 ''' x = cu.spheredist(GC['RAgal'],GC['DECgal'],W['RA'][nyu_id],W['DEC'][nyu_id]) bins = np.arange(0,0.1,da) h, bins = np.histogram(x,bins=bins) bins = bins*3600.0 print sum(h[1:]) print len(GC)-len(result_1[0]) plt.plot(bins[:-1],h[:]) plt.yscale('log') plt.xlabel('da (deg)') plt.ylabel('N') plt.title(catalogue1) plt.show(block=False) ''' filename1 = catalogue2+'_'+catalogue1+'_match' filename2 = catalogue1+'_'+catalogue2+'_match' np.save(savepath1+filename1, result[1].astype(int)) np.save(savepath2+filename2, result[0].astype(int))
def main(): ###make sure to change these when running in a new enviorment!### #location of data directory filepath_cat1 = cu.get_output_path() + 'processed_data/yang_groupcat/' filepath_cat2 = cu.get_output_path() + 'processed_data/mpa_dr7/' #save data to directory... savepath1 = filepath_cat1 + 'mpa_dr7_match/' savepath2 = filepath_cat2 + 'yang_groupcat_match/' ################################################################# catalogues_1=['sample1_L_petro','sample2_L_petro','sample3_L_petro',\ 'sample1_M_petro','sample2_M_petro','sample3_M_petro',\ 'sample1_L_model','sample2_L_model','sample3_L_model',\ 'sample1_M_model','sample2_M_model','sample3_M_model'] catalogues_2 = ['gal_info_gal_totspecsfr_dr7_v5_2'] for catalogue in catalogues_1: catalogue1 = catalogue catalogue2 = catalogues_2[0] print catalogue1, 'match into', catalogue2 f1 = h5py.File(filepath_cat1 + catalogue1 + '.hdf5', 'r') #open catalogue file GC = f1.get(catalogue1) f2 = h5py.File(filepath_cat2 + catalogue2 + '.hdf5', 'r') #open catalogue file W = f2.get(catalogue2) da = 2.0 * 1.0 / 3600.0 #matching length result_1 = np.array( cu.spherematch(GC['RAgal'], GC['DECgal'], W['RA'], W['DEC'], tol=da, nnearest=1)) #result_2 = np.array(cu.spherematch(GC['RAgal'], GC['DECgal'], W['RA'], W['DEC'], tol=da, nnearest=2)) #not used repeats = [ item for item, count in Counter(result_1[1]).iteritems() if count > 1 ] #double matched objects if len(repeats) > 0: remove = np.zeros((0, ), dtype=np.int) #which entries to remove for repeat in repeats: result_a = np.where( result_1[1] == repeat )[0] #find indices of the double matched object into catalogue2 result_b = np.where( result_1[2][result_a] > np.min(result_1[2][result_a]))[ 0] #find less good matches result_c = result_a[ result_b] #indices of less good matches into catalogue2 remove = np.hstack( (remove, result_c)) #indices which should be removed keep = np.arange(0, len(result_1[0]), 1).astype(int) keep = np.in1d(keep, remove) == False result = result_1[:, keep] #unique = np.setdiff1d(result_2[0], result_1[0]) #not used else: result = result filename1 = catalogue2 + '_' + catalogue1 + '_match' filename2 = catalogue1 + '_' + catalogue2 + '_match' np.save(savepath1 + filename1, result[1].astype(int)) np.save(savepath2 + filename2, result[0].astype(int))
def main(): ###make sure to change these when running in a new enviorment!### #location of data directory filepath_cat1 = cu.get_output_path() + 'processed_data/yang_groupcat/' filepath_cat2 = cu.get_output_path() + 'processed_data/NYU_VAGC/' #save data to directory... savepath1 = filepath_cat1 + 'nyu_vagc_match/' savepath2 = filepath_cat2 + 'yang_groupcat_match/' ################################################################# catalogues_1 = ['sample1_L_petro','sample2_L_petro','sample3_L_petro',\ 'sample1_M_petro','sample2_M_petro','sample3_M_petro',\ 'sample1_L_model','sample2_L_model','sample3_L_model',\ 'sample1_M_model','sample2_M_model','sample3_M_model'] catalogues_2 = ['nyu_vagc_dr7'] for catalogue in catalogues_1: catalogue1 = catalogue catalogue2 = catalogues_2[0] print catalogue1, 'match into', catalogue2 f1 = h5py.File(filepath_cat1 + catalogue1 + '.hdf5', 'r') #open catalogue file GC = f1.get(catalogue1) f2 = h5py.File(filepath_cat2 + catalogue2 + '.hdf5', 'r') #open catalogue file W = f2.get(catalogue2) da = 2.0 * 1.0 / 3600.0 #matching length in degrees result_1 = np.array( cu.spherematch(GC['RAgal'], GC['DECgal'], W['RA'], W['DEC'], tol=da, nnearest=1)) #result_2 = np.array(cu.spherematch(GC['RAgal'], GC['DECgal'], W['RA'], W['DEC'], tol=da, nnearest=2)) #not used repeats = [ item for item, count in Counter(result_1[1]).iteritems() if count > 1 ] #double matched objects if len(repeats) > 0: remove = np.zeros((0, ), dtype=np.int) #which entries to remove for repeat in repeats: result_a = np.where( result_1[1] == repeat )[0] #find indices of the double matched object into catalogue2 result_b = np.where( result_1[2][result_a] > np.min(result_1[2][result_a]))[ 0] #find less good matches result_c = result_a[ result_b] #indices of less good matches into catalogue2 remove = np.hstack( (remove, result_c)) #indices which should be removed keep = np.arange(0, len(result_1[0]), 1).astype(int) keep = np.in1d(keep, remove) == False result = result_1[:, keep] #unique = np.setdiff1d(result_2[0], result_1[0]) #not used else: result = result_1 ''' x = cu.spheredist(GC['RAgal'],GC['DECgal'],W['RA'][nyu_id],W['DEC'][nyu_id]) bins = np.arange(0,0.1,da) h, bins = np.histogram(x,bins=bins) bins = bins*3600.0 print sum(h[1:]) print len(GC)-len(result_1[0]) plt.plot(bins[:-1],h[:]) plt.yscale('log') plt.xlabel('da (deg)') plt.ylabel('N') plt.title(catalogue1) plt.show(block=False) ''' filename1 = catalogue2 + '_' + catalogue1 + '_match' filename2 = catalogue1 + '_' + catalogue2 + '_match' np.save(savepath1 + filename1, result[1].astype(int)) np.save(savepath2 + filename2, result[0].astype(int))
def main(): ###make sure to change these when running in a new enviorment!### #location of data directory filepath_cat1 = cu.get_output_path() + 'processed_data/Berland_groupcat/' filepath_cat2 = cu.get_output_path() + 'processed_data/NYU_VAGC/' #save data to directory... savepath1 = filepath_cat1 + 'nyu_vagc_match/' savepath2 = filepath_cat2 + 'berland_groupcat_match/' ################################################################# catalogues_1 = ['mr19_groups', 'smthresh10.2.groups', 'smthresh9.8.groups'] catalogues_2 = ['nyu_vagc_dr7'] catalogue1 = catalogues_1[0] catalogue2 = catalogues_2[0] print catalogue1, 'match into', catalogue2 f1 = h5py.File(filepath_cat1 + catalogue1 + '.hdf5', 'r') #open catalogue file GC = f1.get(catalogue1) f2 = h5py.File(filepath_cat2 + catalogue2 + '.hdf5', 'r') #open catalogue file W = f2.get(catalogue2) da = 2.0 * 1.0 / 3600.0 #matching length result = cu.spherematch(GC['RA'], GC['DEC'], W['RA'], W['DEC'], tol=da, nnearest=1) #check to see if anything was matched to the same object repeats = [ item for item, count in Counter(result[1]).iteritems() if count > 1 ] if len(repeats) > 0: print 'number of double matched objects:', len(repeats) #check to see if every object has a match if len(result[0]) == len(GC): print 'a match was found for every object.' else: print 'some objects do not have matches.' filename1 = catalogue2 + '_' + catalogue1 + '_match' filename2 = catalogue1 + '_' + catalogue2 + '_match' np.save(savepath1 + filename1, result[1]) np.save(savepath2 + filename2, result[0]) catalogue1 = catalogues_1[1] catalogue2 = catalogues_2[0] print catalogue1, 'match into', catalogue2 f1 = h5py.File(filepath_cat1 + catalogue1 + '.hdf5', 'r') #open catalogue file GC = f1.get(catalogue1) f2 = h5py.File(filepath_cat2 + catalogue2 + '.hdf5', 'r') #open catalogue file W = f2.get(catalogue2) da = 2.0 * 1.0 / 3600.0 #matching length result = cu.spherematch(GC['ra'], GC['dec'], W['RA'], W['DEC'], tol=da, nnearest=1) #check to see if anything was matched to the same object repeats = [ item for item, count in Counter(result[1]).iteritems() if count > 1 ] if len(repeats) > 0: print 'number of double matched objects:', len(repeats) #check to see if every object has a match if len(result[0]) == len(GC): print 'a match was found for every object.' else: print 'some objects do not have matches.' filename1 = catalogue2 + '_' + catalogue1 + '_match' filename2 = catalogue1 + '_' + catalogue2 + '_match' np.save(savepath1 + filename1, result[1]) np.save(savepath2 + filename2, result[0]) catalogue1 = catalogues_1[2] catalogue2 = catalogues_2[0] print catalogue1, 'match into', catalogue2 f1 = h5py.File(filepath_cat1 + catalogue1 + '.hdf5', 'r') #open catalogue file GC = f1.get(catalogue1) f2 = h5py.File(filepath_cat2 + catalogue2 + '.hdf5', 'r') #open catalogue file W = f2.get(catalogue2) da = 2.0 * 1.0 / 3600.0 #matching length result = cu.spherematch(GC['ra'], GC['dec'], W['RA'], W['DEC'], tol=da, nnearest=1) #check to see if anything was matched to the same object repeats = [ item for item, count in Counter(result[1]).iteritems() if count > 1 ] if len(repeats) > 0: print 'number of double matched objects:', len(repeats) #check to see if every object has a match if len(result[0]) == len(GC): print 'a match was found for every object.' else: print 'some objects do not have matches.' filename1 = catalogue2 + '_' + catalogue1 + '_match' filename2 = catalogue1 + '_' + catalogue2 + '_match' np.save(savepath1 + filename1, result[1]) np.save(savepath2 + filename2, result[0])
def main(): ###make sure to change these when running in a new enviorment!### #location of data directory filepath_cat1 = cu.get_output_path() + 'processed_data/Berland_groupcat/' filepath_cat2 = cu.get_output_path() + 'processed_data/NYU_VAGC/' #save data to directory... savepath1 = filepath_cat1+'nyu_vagc_match/' savepath2 = filepath_cat2+'berland_groupcat_match/' ################################################################# catalogues_1=['mr19_groups', 'smthresh10.2.groups', 'smthresh9.8.groups'] catalogues_2=['nyu_vagc_dr7'] catalogue1=catalogues_1[0] catalogue2=catalogues_2[0] print catalogue1,'match into', catalogue2 f1 = h5py.File(filepath_cat1+catalogue1+'.hdf5', 'r') #open catalogue file GC = f1.get(catalogue1) f2 = h5py.File(filepath_cat2+catalogue2+'.hdf5', 'r') #open catalogue file W = f2.get(catalogue2) da=2.0*1.0/3600.0 #matching length result = cu.spherematch(GC['RA'], GC['DEC'], W['RA'], W['DEC'], tol=da, nnearest=1) #check to see if anything was matched to the same object repeats = [item for item, count in Counter(result[1]).iteritems() if count > 1] if len(repeats)>0: print 'number of double matched objects:', len(repeats) #check to see if every object has a match if len(result[0])==len(GC): print 'a match was found for every object.' else: print 'some objects do not have matches.' filename1 = catalogue2+'_'+catalogue1+'_match' filename2 = catalogue1+'_'+catalogue2+'_match' np.save(savepath1+filename1, result[1]) np.save(savepath2+filename2, result[0]) catalogue1=catalogues_1[1] catalogue2=catalogues_2[0] print catalogue1,'match into', catalogue2 f1 = h5py.File(filepath_cat1+catalogue1+'.hdf5', 'r') #open catalogue file GC = f1.get(catalogue1) f2 = h5py.File(filepath_cat2+catalogue2+'.hdf5', 'r') #open catalogue file W = f2.get(catalogue2) da=2.0*1.0/3600.0 #matching length result = cu.spherematch(GC['ra'], GC['dec'], W['RA'], W['DEC'], tol=da, nnearest=1) #check to see if anything was matched to the same object repeats = [item for item, count in Counter(result[1]).iteritems() if count > 1] if len(repeats)>0: print 'number of double matched objects:', len(repeats) #check to see if every object has a match if len(result[0])==len(GC): print 'a match was found for every object.' else: print 'some objects do not have matches.' filename1 = catalogue2+'_'+catalogue1+'_match' filename2 = catalogue1+'_'+catalogue2+'_match' np.save(savepath1+filename1, result[1]) np.save(savepath2+filename2, result[0]) catalogue1=catalogues_1[2] catalogue2=catalogues_2[0] print catalogue1,'match into', catalogue2 f1 = h5py.File(filepath_cat1+catalogue1+'.hdf5', 'r') #open catalogue file GC = f1.get(catalogue1) f2 = h5py.File(filepath_cat2+catalogue2+'.hdf5', 'r') #open catalogue file W = f2.get(catalogue2) da=2.0*1.0/3600.0 #matching length result = cu.spherematch(GC['ra'], GC['dec'], W['RA'], W['DEC'], tol=da, nnearest=1) #check to see if anything was matched to the same object repeats = [item for item, count in Counter(result[1]).iteritems() if count > 1] if len(repeats)>0: print 'number of double matched objects:', len(repeats) #check to see if every object has a match if len(result[0])==len(GC): print 'a match was found for every object.' else: print 'some objects do not have matches.' filename1 = catalogue2+'_'+catalogue1+'_match' filename2 = catalogue1+'_'+catalogue2+'_match' np.save(savepath1+filename1, result[1]) np.save(savepath2+filename2, result[0])