def find_center(points, weights, star):
x = [p[0] for p in points]
y = [p[1] for p in points]
c = (np.average(x, weights=weights), np.average(y, weights=weights))
#print w
print c
Ra_shift = astCoords.calcAngSepDeg(c[0],c[1],star[0],c[1])*3600
Dec_shift = astCoords.calcAngSepDeg(c[0],c[1],c[0],star[1])*3600
# Need to shift to the South West, so make the shifts negative
new_coords = astCoords.shiftRADec(c[0], c[1], -1*Ra_shift, -1*Dec_shift)
print Ra_shift, Dec_shift
print new_coords
def indmatch(ra1, dec1, ra2, dec2, tol, one=True):
"""
Finds objects in ra1, dec1 that have a matching object in ra2,dec2
within tol arcsec.
Returns i1, i2 where i1 are indices into ra1,dec1 that have
matches, and i2 are the indices into ra2, dec2 giving the matching
objects.
:param: one, whether one-to-one mapping should be done
"""
m = match(ra1, dec1, ra2, dec2, tol)
c = m.ind > -1
i1 = c.nonzero()[0]
i2 = m.ind[c]
if one:
dl = 0
# :todo: this is horribly written, should do better
for x in i2:
tmp = np.where(i2 == x)[0]
if len(tmp) > 1:
#found a duplicate
keeps = i1[tmp]
rm = i2[tmp][0]
dists = astCoords.calcAngSepDeg(ra2[rm], dec2[rm],
ra1[keeps], dec1[keeps])
smaller = np.argmin(dists)
delete = tmp[tmp != tmp[smaller]]
i1 = np.delete(i1, delete)
i2 = np.delete(i2, delete)
dl += len(delete)
print 'removed %i double matches, left the closest match' % dl
return i1, i2
def separation(ra1, dec1, ra2, dec2, pa, incl, D_gal, wcsheader):
logfile = "rhodump.log"
logdump(logfile, "{0} {1} {2} {3}\n".format(ra1, dec1, ra2, dec2))
#Assuming ra & dec come in as strings as "hh mm ss.sss"
#1. calculate separation angle between the two coordinates, giving us r_raw
#Strip: can't have leading or trailing whitespaces
r1, d1 = astCoords.hms2decimal(strip(ra1), " "), astCoords.dms2decimal(strip(dec1), " ")
r2, d2 = astCoords.hms2decimal(strip(ra2), " "), astCoords.dms2decimal(strip(dec2), " ")
angle = astCoords.calcAngSepDeg(r1, d1, r2, d2)
r_raw = D_gal * m.radians(angle)
#print r_raw, "r_raw"
#2. correct for pa, i by deprojecting radius
#For this we need x and y coordinates through the wcs header
x1, y1 = wcsheader.wcs2pix(r1, d1)
x2, y2 = wcsheader.wcs2pix(r2, d2)
e_a = e_angle_rad(x1, y1, x2, y2, pa) #get the angle
esqd = 1 - (m.cos(m.radians(incl)))**2. #deproject
sqrtf = m.sqrt((1-esqd)/(1-esqd*m.cos(e_a)**2.))
#print esqd, m.cos(e_a), sqrtf, "esqd cos(angle) sqrtf"
#calculate separation in parsec from angle and ellipse deprojection
logdump(logfile, "{0} {1} {2}\n".format(r_raw, sqrtf, r_raw/sqrtf))
#Append to a ds9-style file - replace our ' ' with ':'
r1s = ra1.strip().replace(' ', ':')
d1s = dec1.strip().replace(' ', ':')
r2s = ra2.strip().replace(' ', ':')
d2s = dec2.strip().replace(' ', ':')
logdump("rholog.reg", "circle({0},{1},{2}\")\n".format(r1s, d1s, r_raw/sqrtf/D_gal*3600))
logdump("rholog.reg", "point({0},{1})\n".format(r2s,d2s))
return r_raw / sqrtf #in units of D_gal (e.g. kpc)
def physicalSeparation(inputdata, redshift, H0=71.0, OmegaM=0.28):
"""
Calculates the physical distances between objects of given RAs and DECs at a given redshift.
:param inputdata: coordinates (RA and DEC) of the objects. The input data should
consist of four keys value pairs::
RA1: RA of the object from which to calculate the distance (float)
DEC1: DEC of the object from which to calculate the distance (float)
RA2: RAs of the objects (float or numpy array)
DEC2: DECs of the objects (float or numpy array)
:type inputdata: dict
:param redshift: cosmological redshift of the object
:type redshift: float
:param H0: Hubble value [71/km/s/Mpc]
:type H0: float
:param OmegaM: Matter density
:type OmegaM: float [0.28]
:return: physical distance [distance], scale at a given redshift 1 arcsec = kpc [scale],
separation on the sky in arcseconds [separation], cosmological parameters [cosmology]
:rtype: dict
"""
sep = Coords.calcAngSepDeg(inputdata["RA1"], inputdata["DEC1"], inputdata["RA2"], inputdata["DEC2"])
d = cosmocalc(redshift, H0, OmegaM)
dd = d["PS_kpc"]
pd = angularSeparationToPhysical(sep, dd)
return dict(distance=pd, scale=dd, separation=sep / DEGTOARCSEC, redshift=redshift, cosmology=d)
def _outputMinima(self):
"""
Outputs the results to a file and also to the screen if debugging was turned on.
"""
if 'chi' in self.fitting['method']:
str = '{0:.2f}\t{1:.0f}\t{2:.0f}\t{3:.0f}\t{4:.0f}\t{5:.2f}\t\t{6:.2f}\n'.format(self.result['rotation'],
self.result['xcenter'],
self.result['ycenter'],
self.result['x'],
self.result['y'],
self.result[
'minimaPosition'][3],
self.result[
'minimaPosition'][4])
elif 'corr' in self.fitting['method']:
str = '{0:.2f}\t{1:.0f}\t{2:.0f}\t{3:.0f}\t{4:.0f}\t{5:.5f}\n'.format(self.result['rotation'],
self.result['xcenter'],
self.result['ycenter'],
self.result['x'],
self.result['y'],
self.result['minimaPosition'][6])
else:
raise NotImplementedError, 'This minimization method has not yet been implemented...'
fh1 = open('min.txt', 'a')
fh1.write(str)
fh1.close()
if self.debug:
if 'chi' in self.fitting['method']:
print '\n\nr \t x \t y \txoff \tyoff \tchi**2 reduced chi**2'
elif 'corr' in self.fitting['method']:
print '\n\nr \t x \t y \txoff \tyoff \tcorrelation coefficient'
else:
raise NotImplementedError, 'This minimization method has not yet been implemented...'
print str
print '\nInitial RA and DEC (of the centre of the centre slit)'
print astCoords.decimal2hms(self.result['RAinit'], ':'), astCoords.decimal2dms(self.result['DECinit'], ':')
print '\nFinal RA and DEC (of the centre of the centre slit)'
print astCoords.decimal2hms(self.result['RA'], ':'), astCoords.decimal2dms(self.result['DEC'], ':')
print '\nDistance on the sky (in arcseconds)'
print astCoords.calcAngSepDeg(self.result['RAinit'],
self.result['DECinit'],
self.result['RA'],
self.result['DEC']) * 3600
def imagePositionFilter(self,ra,dec,radius,units="arcminutes"):
"""
Filters a queryset based on arclength. Returns a List of
database rows, therefore must be the last piece of a QuerySet chain.
"""
convert_units = {'arcminutes':60.,'arcseconds':3600.,'degrees':1}
radius /= convert_units[units]
results = []
for i in self:
if astCoords.calcAngSepDeg(i.CRVAL1,i.CRVAL2,ra,dec) <= radius:
results.append(i)
return results
def positionFilter(self,ra,dec,radius,units="arcminutes"):
"""
Filters a queryset based on arclength. Returns a List of
database rows, therefore must be the last piece of a QuerySet chain.
"""
radius *= constants.convert_arcmin_or_arcsec_to_degrees[units]
results = []
for i in self.filter(RA__range=(ra-radius,ra+radius),DEC__range=(dec-radius,dec+radius)):
distance = astCoords.calcAngSepDeg(i.RA,i.DEC,ra,dec)
if distance <= radius:
i.__setattr__("distance",distance)
results.append(i)
return results
def findSeperationSpatial(data, center):
''' Finds the distance to all of the galaxies from the center of the
cluster in the spatial plane. Returns values in Mpc.
'''
# Add a new column to the dataframe
data['seperation'] = 0.0
for row in data.iterrows():
sepDeg = aco.calcAngSepDeg(center[0], center[1], row[1]['ra'],
row[1]['dec'])
sepMpc = sepDeg * aca.da(row[1]['redshift']) / 57.2957795131
data['seperation'][row[0]] = sepMpc
return data
def findseparationSpatial(data, center):
''' Finds the distance to all of the galaxies from the center of the
cluster in the spatial plane. Returns values in Mpc.
'''
# Add a new column to the dataframe
sepDeg = np.array(aco.calcAngSepDeg(center[0], center[1], data.ra.values,
data.dec.values))
da = np.array([aca.da(z) / 57.2957795131 for z in data.redshift.values])
sepMpc = da * sepDeg
data.loc[:, 'separation'] = sepMpc
return data
def _findValues(data, group=3):
"""
Finds redshift and physical distance from raw data.
Data are first grouped by group=3 column
"""
conversion = 0.000277777778 # degree to arcsecond
ddist = dist.getDiameterDistances(data)
redshift = []
pdist = []
mhalo = []
for x in set(data[:, group]):
#find all galaxies
tmp = data[data[:, group] == x]
if len(tmp) > 1:
#redshift
z = tmp[0][0]
#look the diameter distance from the lookup table
dd = ddist[z]
#the first halo, assume it to be the main halo
RADeg1 = tmp[0][1]
decDeg1 = tmp[0][2]
#the following haloes, assume to be subhaloes
RADeg2 = tmp[1:, 1]
decDeg2 = tmp[1:, 2]
#calculate the angular separation on the sky
sep = Coords.calcAngSepDeg(RADeg1, decDeg1, RADeg2, decDeg2)
#convert to physical distance on that redshift
physical_distance = sep * dd / conversion
#append the results
redshift.append(z)
pdist.append(physical_distance)
mhalo.append(tmp[0][5])
dict = {'redshift': redshift, 'physical_distance': pdist, 'mhalo': mhalo}
return dict
def correlate (array1, ra1, dec1, array2, ra2, dec2, dist, \
mindist=0.0, isabs=False, noisy=True):
if not have_astLib:
print 'No astLib, exiting'
return []
fstart = nfstart = 0
fend = array2.shape[0]
icou = 0
correl = np.array([])
decfac=1.0 if isabs else min(1./np.cos(np.deg2rad(array1[:,dec1].max())),\
1./np.cos(np.deg2rad(array2[:,dec2].max())))
for i in range(array1.shape[0]):
i10 = np.linspace(0, array1.shape[0], 10, dtype='int')
i100 = np.linspace(0, array1.shape[0], 100, dtype='int')
if i in i10 and noisy:
sys.stdout.write('*')
sys.stdout.flush()
elif i in i100 and noisy:
sys.stdout.write('.')
sys.stdout.flush()
else:
pass
fstart = nfstart
for j in range(fstart, fend):
radiff = array2[j, ra2] - array1[i, ra1]
if radiff < -decfac * dist:
nfstart = j
if radiff > decfac * dist:
break
if abs(array2[j, dec2] - array1[i, dec1]) > dist:
continue
adist = np.hypot(array1[i,ra1]-array2[j,ra2],\
array1[i,dec1]-array2[j,dec2]) if isabs \
else astCoords.calcAngSepDeg(array1[i,ra1],array1[i,dec1],\
array2[j,ra2],array2[j,dec2])
if adist < dist and abs(radiff) < 90.0 and adist >= mindist:
try:
correl = np.vstack((correl, np.array([i, j, adist])))
except:
correl = np.array([[i, j, adist]])
return correl
def run(center, min_Nperbin=15, min_binsize=250,
maingap=500, maxgap=1000, sigma=False, version='1'):
#center = centers[i]
id = center[0]
xo = center[1]
yo = center[2]
zo = center[3]
j = close(xo, yo, zo, ra, dec, z)
r = astCoords.calcAngSepDeg(xo, yo, ra[j], dec[j])
r = scipy.array([cosmology.dProj(zo, ri, input_unit='deg', unit='Mpc') \
for ri in r])
if '2' in version:
r *= (1 + zo)
r *= 1e3
outplot = get_plotname(version, id)
#output = 'phase%s/plots/v%s/rv/halo-%d.png' %(phase, version, id)
m = members.sgapper(r, z[j], zo=zo, min_Nperbin=min_Nperbin,
maingap=maingap, maxgap=maxgap, sigma=sigma,
verbose=False, debug=False, plot_output=outplot,
full_output=True)
m, binsize, nit, incut = m
Nm = len(m)
zo = stattools.Cbi(z[j[m]])
s, m200, r200 = mass(z[j[m]], zo)
mo = m[r[m] <= r200]
n200 = -1
it = []
while n200 != len(mo):
try:
n200 = len(mo)
s, m200, r200 = mass(z[j[mo]], zo)
#zo = stattools.Cbi(z[j[mo]])
mo = m[r[m] <= r200]
# should maybe find a more sophisticated solution by, say, averaging
# all items from one of these "cycles"
if r200 in it:
break
it.append(r200)
except ZeroDivisionError:
print 'broke'
break
if n200 > 15:
stats = (stattools.Cbi, stattools.Sbi)
z_err, s_err = stattools.bootstrap(stats, z[j[incut]],
n_obj=Nm, n_samples=1000)
s_err = c * s_err / (1+zo)
else:
z_err = scipy.std(z) / scipy.sqrt(Nm)
s_err = c/(1+zo) * z_err / scipy.sqrt(2)
m200, m200_err = scalings.sigma(s, zo, s_err, z_err,
separate_errors=True)
r200 = conversions.rsph(m200, zo)
# to make it their format
m200_err = [scipy.log10(1+me/m200) for me in m200_err]
try:
#if n200 >
line = '%4d %.3f %.3f %6d %.1e %4d %.1f %4d %.2f %.2f %4d' \
%(id, xo*deg2rad, yo*deg2rad, round(c*zo, 0), m200,
round(s, 0), r200/1e3, n200, m200_err[0], m200_err[1], Nm)
except TypeError:
line = '%4d %.3f %.3f %6d -1 -1 -1 -1 -1 -1 %4d' \
%(id, xo*deg2rad, yo*deg2rad, round(c*zo, 0), Nm)
print line, '%4.1f' %max(r[m]/1e3)
out = open(output, 'a')
print >>out, line
out.close()
write_members(id, galid[j[m]], galid[j[mo]])
return m200
#group them by halo_id
for x in set(matched[:, 3]):
tmp = matched[matched[:, 3] == x]
#redshift
z = tmp[0][0]
RADeg1 = tmp[0][1]
decDeg1 = tmp[0][2]
RADeg2 = tmp[1:, 1]
decDeg2 = tmp[1:, 2]
prop = tmp[:, 5]
sep = Coords.calcAngSepDeg(RADeg1, decDeg1, RADeg2, decDeg2)
dd = cosmocalc(z, 71.0,
0.28)['PS_kpc'] #dist.getDiameterDistances(data)
physical_distance = (sep / dd) / conversion
print z, RADeg1, RADeg2, decDeg1, decDeg2, prop
print sep, physical_distance
print
ax.plot([z for foo in range(len(physical_distance))],
physical_distance, 'bo')
#P.show()
print
# our info
table['RA'] = nan
table['DEC'] = nan
table['Dist'] = nan
table['z'] = m['zBCG_boada']
table['Mag Lim'] = m['Cmag_i']
#table['N'] = nan
# extern info
table['RA EX'] = m['RA_y']
table['DEC EX'] = m['DEC_y']
table['Dist EX'] = nan
table['z EX'] = m['z_extern']
#table['N EX'] = m['R']
table['Flag'] = m['Flag']
table['Ref'] = m['REDSHIFT_SOURCE']
fixed1 = m.loc[pd.notnull(m['RAJ2000']),
'RAJ2000'].apply(lambda x: astCoords.decimal2hms(x, ':'))
fixed2 = m.loc[pd.notnull(m['DEJ2000']),
'DEJ2000'].apply(lambda x: astCoords.decimal2dms(x, ':'))
table.loc[fixed1.index, 'RA EX'] = fixed1
table.loc[fixed2.index, 'DEC EX'] = fixed2
for i, row in table.iterrows():
table.loc[i, 'Dist EX'] = astCoords.calcAngSepDeg(
row['RA PSZ'], row['DEC PSZ'], row['RA EX'], row['DEC EX']) * 60
print('done')
if zunit = 'velocity':
v = z
zo /= c
else:
v = c * (z-zo)/(1+zo)
# then x corresponds to cluster-centric distance:
if len(y) == 0:
if xyunit == 'kpc':
r = x / 1e3
elif xyunit in ('deg', 'arcmin', 'arcsec'):
r = cosmology.dProj(zo, x, input_unit=xyunit, unit='Mpc')
# otherwise use the given center to calculate distances
elif xyunit in ('kpc', 'Mpc'):
r = scipy.hypot(x, y)
if xyunit == 'kpc':
r /= 1e3
else:
if xyunit == 'arcsec':
x /= 3600.
y /= 3600.
if xyunit == 'arcmin':
x /= 60.
y /= 60.
dist = astCoords.calcAngSepDeg(x, y, xycenter[0], xycenter[1])
r = cosmology.dProj(zo, dist, input_unit='deg', unit='Mpc')
return
### THIS IS WHERE OBJECTS ARE FOUND FROM TARGETS.list
### ... THEY MUST BE WITHIN 0.04deg (2.4") OF THE TARGETS.list VALUE
for raw_dir in raw_dirs:
targetstxt = open("targets.txt", "w")
for target in targets:
made_dir = False
target_ra = astCoords.hms2decimal(target[1], ":")
target_dec = astCoords.dms2decimal(target[2], ":")
for im in raw_dir.obj:
obj_ra = astCoords.hms2decimal(pf.getval(working_dir + "/raw/"
+ raw_dir.name + "/"
+ im, "RA"), ":")
obj_dec = astCoords.dms2decimal(pf.getval(working_dir + "/raw/"
+ raw_dir.name + "/"
+ im, "DEC"), ":")
if astCoords.calcAngSepDeg(target_ra, target_dec,
obj_ra, obj_dec) < 0.04:
if not made_dir:
os.mkdir(raw_dir.name+"/"+target[0])
made_dir = True
targetstxt.write(target[0]+"\n")
os.chdir(raw_dir.name + "/" + target[0])
ln_cmd = "ln -s " + working_dir + "/raw/" \
+ raw_dir.name + "/" + im + " ./"
sp.call( [ln_cmd], shell=True )
os.chdir(working_dir + '/redux/')
targetstxt.close()
shu.move("targets.txt", raw_dir.name)
for raw_dir in raw_dirs:
os.mkdir(raw_dir.name + "/calib/")
def query(ra, dec, radius=2., unit='arcmin', z=0., cosmo=None,
catalogs=None, return_single=True, squeeze=False,
return_values=('name','ra','dec','z','index','dist','dz')):
"""
Query different catalogs for clusters close to a given set of coordinates.
To-do's:
-possibility to return survey observables
Parameters
----------
ra : (array of) float or str
if float, should be the RA in decimal degrees; if str,
should be in hms format ('hh:mm:ss', requires astLib)
dec : (array of) float or str
if float, should be the Dec in decimal degrees; if str,
should be in dms format ('dd:mm:ss', requires astLib)
radius : float (default 2)
the search radius, in units given by argumetn "unit"
unit : {'arcmin', 'Mpc'}
the units of argument "radius". If Mpc, then argument "z"
must be larger than zero.
z : (array of) float (optional)
redshift(s) of the cluster(s). Must be >0 if unit=='Mpc'.
cosmo : module astro.cosmology (optional)
if the matching is done in physical distance then pass
this module to make sure all cosmological parameters are
used consistently with the parent code!
catalogs : str (optional)
list or comma-separated names of catalogs to be searched.
If not given, all available catalogs are searched. Allowed
values are:
* 'maxbcg' (Koester et al. 2007)
* 'gmbcg' (Hao et al. 2010)
* 'hecs2013' (Rines et al. 2013)
* 'hecs2016' (Rines et al. 2016) NOT YET
* 'orca' (Geach, Murphy & Bower 2011)
* 'psz1' (Planck Collaboration XXIX 2014)
* 'psz2' (Planck Collaboration XXVII 2016)
* 'redmapper' (Rykoff et al. 2014, v5.10)
* 'whl' (Wen, Han & Liu 2012, Wen & Han 2015)
return_single : bool
whether to return the single closest matching cluster (if
within the search radius) or all those falling within the
search radius
return_values : any subset of ('name','ra','dec','z','index','dist','dz')
what elements to return. 'index', 'dist' and 'dz' refer to
the index in the catalog and the distances of the matching
cluster(s) in arcmin and redshift space, respectively.
NOTE: altering the order of the elements in return_values
will have no effect in the order in which they are returned!
squeeze : bool
whether to return a list instead of a dictionary if only
one catalog is provided
Returns
-------
matches : dict
matching elements per catalog. Each requested catalog is
a key of this dictionary if more than one catalog was
searched or if squeeze==False. If only one catalog was
provided and squeeze==True, then return a list with
matching entry/ies.
withmatch : dict
for each searched catalog, contains hte indices of the
provided clusters for which at least one match was found.
The same formatting as for "matches" applies.
"""
available = ('maxbcg', 'gmbcg', 'hecs2013', 'orca', 'psz1', 'psz2',
'redmapper', 'whl')
# some formatting for convenience
if not hasattr(ra, '__iter__'):
ra = [ra]
dec = [dec]
if not hasattr(z, '__iter__'):
z = array([z])
# in the case of matching by physical radius, demand z > 0
if unit == 'Mpc' and npany(z <= 0):
msg = "ERROR: in catalogs.query:"
msg += " if unit=='Mpc' then z must be larger than 0"
print msg
exit()
if unit == 'Mpc':
if cosmo is None:
cosmo = cosmology
dproj = cosmo.dProj
# will this fail for numpy.string_?
if isinstance(ra[0], basestring):
ra = array([hms2decimal(x, ':') for x in ra])
dec = array([dms2decimal(y, ':') for y in dec])
if unit == 'arcmin':
radius = ones(ra.size) * radius# / 60
else:
radius = array([dproj(zi, radius, input_unit='Mpc', unit='arcmin')
for zi in z])
if catalogs is None:
catalogs = available
else:
try:
catalogs = catalogs.split(',')
# if this happens then we assume catalogs is already a list
except ValueError:
pass
for name in catalogs:
if name not in available:
msg = 'WARNING: catalog {0} not available'.format(name)
print msg
labels = {'maxbcg': 'maxBCG', 'gmbcg': 'GMBCG', 'hecs2013': 'HeCS',
'hecs2016': 'HeCS-SZ', 'orca': 'ORCA', 'psz1': 'PSZ1',
'psz2': 'PSZ2', 'redmapper': 'redMaPPer', 'whl': 'WHL'}
filenames = {'maxbcg': 'maxbcg/maxBCG.fits',
'gmbcg': 'gmbcg/GMBCG_SDSS_DR7_PUB.fit',
'hecs2013': 'hecs/2013/data.fits',
'orca': 'orca/fullstripe82.fits',
'psz1': osjoin('planck', 'PSZ-2013', 'PLCK-DR1-SZ',
'COM_PCCS_SZ-union_R1.11.fits'),
'psz2': osjoin('planck', 'PSZ-2015',
'HFI_PCCS_SZ-union_R2.08.fits'),
'redmapper': 'redmapper/redmapper_dr8_public' + \
'_v5.10_catalog.fits',
'whl': 'whl/whl2015.fits'}
columns = {'maxbcg': 'none,RAJ2000,DEJ2000,zph',
'gmbcg': 'OBJID,RA,DEC,PHOTOZ',
'hecs2013': 'Name,RAJ2000,DEJ2000,z',
'orca': 'ID,ra_bcg,dec_bcg,redshift',
'psz1': 'NAME,RA,DEC,REDSHIFT',
'psz2': 'NAME,RA,DEC,REDSHIFT',
'redmapper': 'NAME,RA,DEC,Z_LAMBDA',
'whl': 'WHL,RAJ2000,DEJ2000,zph'}
for cat in catalogs:
filenames[cat] = osjoin(path, filenames[cat])
columns[cat] = columns[cat].split(',')
matches = {}
withmatch = {}
for cat in available:
if cat not in catalogs:
continue
data = getdata(filenames[cat], ext=1)
aux = {}
for name in data.names:
aux[name] = data[name]
data = aux
# if the catalog doesn't give a name
if columns[cat][0] == 'none':
columns[cat][0] = 'Name'
data['Name'] = chararray(data[columns[cat][1]].size, itemsize=4)
data['Name'][:] = 'none'
data = [data[v] for v in columns[cat]]
name, xcat, ycat, zcat = data
colnames = 'name,ra,dec,z'.split(',')
close = [(abs(xcat - x) < 2*r/60.) & (abs(ycat - y) < 2*r/60.)
for x, y, r in izip(ra, dec, radius)]
withmatch[cat] = [j for j, c in enumerate(close) if name[c].size]
dist = [60 * calcAngSepDeg(xcat[j], ycat[j], x, y)
for j, x, y in izip(close, ra, dec)]
match = [(d <= r) for d, r in izip(dist, radius)]
withmatch[cat] = array([w for w, m in izip(count(), match)
if w in withmatch[cat] and name[m].size])
if return_single:
match = [argmin(d) if d.size else None for d in dist]
matches[cat] = {}
# keeping them all now because they may be needed for other properties
for name, x in izip(colnames, data):
matches[cat][name] = array([x[j][mj] for w, j, mj
in izip(count(), close, match)
if w in withmatch[cat]])
if 'index' in return_values:
matches[cat]['index'] = array([arange(xcat.size)[j][m]
for w, j, m in izip(count(),
close, match)
if w in withmatch[cat]])
if 'dist' in return_values:
matches[cat]['dist'] = array([d[m] for w, d, m
in izip(count(), dist, match)
if w in withmatch[cat]])
if unit == 'Mpc':
matches[cat]['dist'] *= array([dproj(zi, 1, unit='Mpc',
input_unit='arcmin')
for zi in matches[cat]['z']])
if 'dz' in return_values:
matches[cat]['dz'] = array([zcat[j][m] - zj for w, j, m, zj
in izip(count(), close, match, z)
if w in withmatch[cat]])
for key in matches[cat].keys():
if key not in return_values:
matches[cat].pop(key)
if not return_single and name[j][match].size == 1:
for key in matches[cat].keys():
matches[cat][key] = matches[cat][key][0]
if len(catalogs) == 1 and squeeze:
return matches[catalogs[0]], withmatch[catalogs[0]]
return matches, withmatch
print zo
# read BCG coordinates
bcg = open(filename).readline().split()[1:]
bcg = [float(x) for x in bcg]
data = readfile.table(filename,
cols=(0, 1, 2, 3),
dtype=(int, float, float, float))
obj = data[0]
ra = data[1]
dec = data[2]
z = data[3]
Ngal = len(obj)
dist = astCoords.calcAngSepDeg(bcg[0], bcg[1], ra, dec)
r = 2 * 1e3 * scipy.array(map(cosmology.dProj, zo * scipy.ones(Ngal), dist))
m = members.sgapper(r,
z,
maingap=500,
min_binsize=250,
converge=False,
plot_output=True,
full_output=True)
m, binsize, nit, incut = m
binloc = [sum(binsize[:i + 1]) for i in xrange(len(binsize))]
c = 299792.458
zo = stattools.Cbi(z[m])
v = c * (z - zo) / (1 + zo)
def testcalcAngSepDeg(self):
for ra1, dec1, ra2, dec2, result in self.calcAngSepDeg:
answer = astCoords.calcAngSepDeg(ra1, dec1, ra2, dec2)
self.assertEqual(result, answer)
def main(round=3):
''' This creates a simple latex table with the results using the columns
specified below. A few things will need to be done by hand after this is
created. The corrected errors to redshifts, and corrected Ngals will need
to be manually added by looking at the results images. The cluster finder
doesn't save those columns by default.
'''
# the confirmed = True gets the 12 confirmed clusters
results = loadClusters(round=round, confirmed=True)
# load the master spreadsheet
t_ss = Table.read('../catalogs/PSZ2_unconfirmed_catalog - current.csv')
df_ss = t_ss.to_pandas()
observed = df_ss.loc[~df_ss['MOSAIC Imaging'].isnull()]
confirmed = observed.merge(results, left_on='Name', right_on='Cluster',
how='left')
# load the PSZ1 catalog
t_1 = Table.read('../catalogs/PSZ1v2.1.fits')
df1 = t_1.to_pandas()
# now we add all of the extra info
# Berrena paper
Bpaper = Table.read('../papers/1803.05764/Barrena_tbl3.csv')
df_paper = Bpaper.to_pandas()
complete = confirmed.merge(df_paper, left_on='Name', right_on='Planck Name',
how='left')
# tack on the PSZ1 catalog
complete = complete.merge(df1, left_on='PSZ1 Indx', right_on='INDEX',
how='left')
# combine some columns to get extra info
complete['z_extern'] = complete['PSZ1 Redshift'].fillna(complete['z_cl'])
complete['S/N'] = complete['SNR_PSZ2'].fillna(complete['SNR_PSZ1'])
# get the columns we want
cols = ['Name', 'PSZ1 Indx', 'PSZ2 Indx', 'S/N', 'RA_SEX', 'DEC_SEX',
'dist_BCG', 'z_cl_boada', 'z_clerr_boada', 'Ngal_c_boada', 'Cmag_i',
'z_extern', 'REDSHIFT_SOURCE', 'BCG_boada']
c = complete.loc[:, cols]
c.loc[:, ('RA_SEX', 'DEC_SEX')] = nan
c.loc[(~c['z_extern'].isnull()) & (c['REDSHIFT_SOURCE'] == -1.0),
'REDSHIFT_SOURCE'] = 99
# let's the the RA/DEC of our BCGs
m = c.loc[c['BCG_boada'].notnull()]
for i, row in m.iterrows():
mems = loadMembers('boada', row['Name'], round=round)
try:
ra = mems.loc[mems['ID'] == row['BCG_boada'], 'RA'].values[0]
except IndexError:
continue
dec = mems.loc[mems['ID'] == row['BCG_boada'], 'DEC'].values[0]
# convert to sexigesimal
ra_sex = astCoords.decimal2hms(ra, ':')
dec_sex = astCoords.decimal2dms(dec, ':')
# compute the distance to the PSZ position
psz_ra = complete.iloc[i]['RA_x']
psz_dec = complete.iloc[i]['DEC_x']
m.loc[i, 'dist_BCG'] = astCoords.calcAngSepDeg(psz_ra,
psz_dec, ra, dec) * 60
# write it back into the main frame
m.loc[i, 'RA_SEX'] = ra_sex
m.loc[i, 'DEC_SEX'] = dec_sex
return m
def match(ra1, dec1, ra2, dec2, tol, allmatches=False):
"""
Given two sets of numpy arrays of ra,dec and a tolerance tol
(float), returns an array of integers with the same length as the
first input array. If integer > 0, it is the index of the closest
matching second array element within tol arcsec. If -1, then there
was no matching ra/dec within tol arcsec.
if allmatches = True, then for each object in the first array,
return the index of everything in the second arrays within the
search tolerance, not just the closest match.
:note: does not force one-to-one mapping
Note to get the indices of objects in ra2, dec2 without a match:
imatch = match(ra1, dec1, ra2, dec2, 2.)
inomatch = numpy.setdiff1d(np.arange(len(ra2)), set(imatch))
"""
DEG_PER_HR = 360. / 24. # degrees per hour
DEG_PER_MIN = DEG_PER_HR / 60. # degrees per min
DEG_PER_S = DEG_PER_MIN / 60. # degrees per sec
DEG_PER_AMIN = 1. / 60. # degrees per arcmin
DEG_PER_ASEC = DEG_PER_AMIN / 60. # degrees per arcsec
RAD_PER_DEG = math.pi / 180. # radians per degree
isorted = ra2.argsort()
sdec2 = dec2[isorted]
sra2 = ra2[isorted]
LIM = tol * DEG_PER_ASEC
match = []
#this is faster but less accurate
# use mean dec, assumes decs similar
#decav = np.mean(sdec2.mean() + dec1.mean())
#RA_LIM = LIM / np.cos(decav * RAD_PER_DEG)
for ra, dec in zip(ra1, dec1):
#slower but more accurate
RA_LIM = LIM / np.cos(dec * RAD_PER_DEG)
i1 = sra2.searchsorted(ra - RA_LIM)
i2 = i1 + sra2[i1:].searchsorted(ra + RA_LIM)
close = []
for j in xrange(i1, i2):
decdist = np.abs(dec - sdec2[j])
if decdist > LIM:
continue
else:
# if ras and decs are within LIM, then
# calculate actual separation
disq = astCoords.calcAngSepDeg(ra, dec, sra2[j], sdec2[j])
close.append((disq, j))
close.sort()
if not allmatches:
# Choose the object with the closest separation inside the
# requested tolerance, if one was found.
if len(close) > 0:
min_dist, jmin = close[0]
if min_dist < LIM:
match.append((isorted[jmin], min_dist))
continue
# otherwise no match
match.append((-1, -1))
else:
# append all the matching objects
jclose = []
seps = []
for dist, j in close:
if dist < LIM:
jclose.append(j)
seps.append(dist)
else:
break
match.append(
fromarrays([isorted[jclose], seps],
dtype=[('ind', 'i8'), ('sep', 'f8')]))
if not allmatches:
# return both indices and separations in a recarray
temp = np.rec.fromrecords(match, names='ind,sep')
# change to arcseconds
temp.sep *= 3600.
temp.sep[temp.sep < 0] = -1.
return temp
else:
return match
def distances(ra, dec, center, z):
d = astCoords.calcAngSepDeg(ra, dec, center[0], center[1])
r = scipy.array([cosmology.dProj(z, di) for di in d])
return 60 * d, r
def match(ra1, dec1, ra2, dec2, tol, allmatches=False):
"""
Given two sets of numpy arrays of ra,dec and a tolerance tol
(float), returns an array of integers with the same length as the
first input array. If integer > 0, it is the index of the closest
matching second array element within tol arcsec. If -1, then there
was no matching ra/dec within tol arcsec.
if allmatches = True, then for each object in the first array,
return the index of everything in the second arrays within the
search tolerance, not just the closest match.
:note: does not force one-to-one mapping
Note to get the indices of objects in ra2, dec2 without a match:
imatch = match(ra1, dec1, ra2, dec2, 2.)
inomatch = numpy.setdiff1d(np.arange(len(ra2)), set(imatch))
"""
DEG_PER_HR = 360. / 24. # degrees per hour
DEG_PER_MIN = DEG_PER_HR / 60. # degrees per min
DEG_PER_S = DEG_PER_MIN / 60. # degrees per sec
DEG_PER_AMIN = 1. / 60. # degrees per arcmin
DEG_PER_ASEC = DEG_PER_AMIN / 60. # degrees per arcsec
RAD_PER_DEG = math.pi / 180. # radians per degree
isorted = ra2.argsort()
sdec2 = dec2[isorted]
sra2 = ra2[isorted]
LIM = tol * DEG_PER_ASEC
match = []
#this is faster but less accurate
# use mean dec, assumes decs similar
#decav = np.mean(sdec2.mean() + dec1.mean())
#RA_LIM = LIM / np.cos(decav * RAD_PER_DEG)
for ra, dec in zip(ra1, dec1):
#slower but more accurate
RA_LIM = LIM / np.cos(dec * RAD_PER_DEG)
i1 = sra2.searchsorted(ra - RA_LIM)
i2 = i1 + sra2[i1:].searchsorted(ra + RA_LIM)
close = []
for j in xrange(i1, i2):
decdist = np.abs(dec - sdec2[j])
if decdist > LIM:
continue
else:
# if ras and decs are within LIM, then
# calculate actual separation
disq = astCoords.calcAngSepDeg(ra, dec, sra2[j], sdec2[j])
close.append((disq, j))
close.sort()
if not allmatches:
# Choose the object with the closest separation inside the
# requested tolerance, if one was found.
if len(close) > 0:
min_dist, jmin = close[0]
if min_dist < LIM:
match.append((isorted[jmin], min_dist))
continue
# otherwise no match
match.append((-1, -1))
else:
# append all the matching objects
jclose = []
seps = []
for dist, j in close:
if dist < LIM:
jclose.append(j)
seps.append(dist)
else:
break
match.append(fromarrays([isorted[jclose], seps],
dtype=[('ind', 'i8'), ('sep', 'f8')]))
if not allmatches:
# return both indices and separations in a recarray
temp = np.rec.fromrecords(match, names='ind,sep')
# change to arcseconds
temp.sep *= 3600.
temp.sep[temp.sep < 0] = -1.
return temp
else:
return match
#group them by halo_id
for x in set(matched[:, 3]):
tmp = matched[matched[:, 3] == x]
#redshift
z = tmp[0][0]
RADeg1 = tmp[0][1]
decDeg1 = tmp[0][2]
RADeg2 = tmp[1:, 1]
decDeg2 = tmp[1:, 2]
prop = tmp[:, 5]
sep = Coords.calcAngSepDeg(RADeg1, decDeg1, RADeg2, decDeg2)
dd = cosmocalc(z, 71.0, 0.28)['PS_kpc'] #dist.getDiameterDistances(data)
physical_distance = (sep / dd) / conversion
print z, RADeg1, RADeg2, decDeg1, decDeg2, prop
print sep, physical_distance
print
ax.plot([z for foo in range(len(physical_distance))], physical_distance, 'bo')
#P.show()
print
print
def Near(v, ra, dec, x, y, stat, Nnear):
d = astCoords.calcAngSepDeg(ra, dec, x, y)
dsort = numpy.argsort(d)
vgroup = v[dsort[1:Nnear+1]]
return stat(vgroup), numpy.std(vgroup)
def getDataSlow(db='database.db', path='/Users/sammy/Research/CANDELS/v2/'):
"""
Get data from the lcone table in db.
:param db: name of the SQLite database file.
:type db: string
:param path: full path of the database file
:type path: string
:Warning: This is extremely slow way to pull out data from a database.
Should be rewritten using table joins etc. so that less queries
could be performed.
:return: redshift, halo mass and radius, projected distances of subhalo galaxies [kpc],
halo_id of the main halo
:rtype: dict
"""
conversion = 0.000277777778 # degree to arcsecond
redshifts = [
'redshift < 0.5 and', 'redshift >= 0.5 and redshift < 1.0 and',
'redshift >= 1.0 and redshift < 2.0 and',
'redshift >= 2.0 and redshift < 3.0 and',
'redshift >= 3.0 and redshift < 4.0 and',
'redshift >= 4.0 and redshift < 5.0 and',
'redshift >= 5.0 and redshift < 6.0 and',
'redshift >= 6.0 and redshift < 7.0 and'
]
for i, red in enumerate(redshifts):
print red
qr = 'select halo_id from lcone where %s gal_id > 1' % red
#qr = 'select halo_id from lcone where %s mhalo > 12.7' % red
#pull out data
ids = sq.get_data_sqliteSMNfunctions(path, db, qr)
uids = np.unique(ids)
print len(uids), 'haloes'
saveid = []
saveorig = []
savedist = []
savemhalo = []
saverhalo = []
saveredshift = []
#we should now look for each unique id
for id in uids:
query = 'select redshift, ra, dec, halo_id, mhalo, rhalo from lcone where halo_id = {0:d}'.format(
id)
#print query
data = sq.get_data_sqliteSMNfunctions(path, db, query)
#if multiples, then don't take it
if len(set(data[:, 0])) > 1:
print 'skipping', id
continue
if len(data[:, 1]) < 2:
print 'no subhaloes', id, data[:, 1]
continue
#redshift
z = data[0, 0]
#look the diameter distance from the lookup table
dd = cosmocalc(z, 71.0, 0.28)['PS_kpc']
#the first halo, assume it to be the main halo
RADeg1 = data[0, 1]
decDeg1 = data[0, 2]
#the following haloes, assume to be subhaloes
RADeg2 = data[1:, 1]
decDeg2 = data[1:, 2]
#calculate the angular separation on the sky
sep = Coords.calcAngSepDeg(RADeg1, decDeg1, RADeg2, decDeg2)
physical_distance = sep * dd / conversion
#these are all the main halo parameters
saveredshift.append(z)
saveid.append(int(data[0, 3]))
savemhalo.append(data[0, 4])
saverhalo.append(data[0, 5])
savedist.append(physical_distance)
saveorig.append(id)
out = dict(halo_ids=saveid,
distances=savedist,
original=saveorig,
mhalo=savemhalo,
rhalo=saverhalo,
redshift=saveredshift)
wr.cPickleDumpDictionary(out, 'distances%i.pickle' % (i + 1))
def quickPlot(galaxies=170, rvir=0.105):
"""
A single halo test
:param galaxies: number of subhalo galaxies
:param rvir: radius of the main halo [Mpc]
"""
conversion = 0.000277777778 # degree to arcsecond
#redshift
z = 4.477
#get the angular diameter distance to the galaxy
dd = cosmocalc(z, 71.0, 0.28)['PS_kpc'] #in kpc / arc seconds
#position of the main galaxy
RADeg1 = 189.41432
decDeg1 = 62.166702
# get the random values
rds = rand.randomUnitSphere(galaxies)
# convert them to Cartesian coord
rd = conv.convertSphericalToCartesian(rvir / dd / 2., rds['theta'],
rds['phi'])
data = {
'CD': np.matrix('-1 0; 0 1'),
'RA': RADeg1,
'DEC': decDeg1,
'X': rd['y'],
'Y': rd['z']
}
result = conv.RAandDECfromStandardCoordinates(data)
RADeg2 = result['RA']
decDeg2 = result['DEC']
#calculate the angular separation on the sky
sep = Coords.calcAngSepDeg(RADeg1, decDeg1, RADeg2, decDeg2) / 2.
physical_distance = sep * dd / conversion
text = r'Random Positions, z={0:.2f}, radius of the halo = {1:.0f} kpc'.format(
z, 1e3 * rvir)
#make a figure
fig = plt.figure(figsize=(12, 12))
ax1 = fig.add_subplot(221)
ax2 = fig.add_subplot(222)
ax3 = fig.add_subplot(223)
rect = fig.add_subplot(224, visible=False).get_position()
ax4 = Axes3D(fig, rect)
ax1.hist(sep * 1e4, bins=12)
ax2.hist(physical_distance, bins=12)
ax3.scatter(0, 0, c='k', s=120, marker='s')
ax3.scatter((RADeg2 - RADeg1) * 1e4, (decDeg2 - decDeg1) * 1e4,
c='b',
s=50,
marker='o',
alpha=0.45)
#generate a sphere
u = np.linspace(0, 2 * np.pi, 100)
v = np.linspace(0, np.pi, 100)
x = np.outer(np.cos(u), np.sin(v))
y = np.outer(np.sin(u), np.sin(v))
z = np.outer(np.ones(np.size(u)), np.cos(v))
#plot it as a wire frame
ax4.plot_wireframe(x * 1e4 * rvir / dd,
y * 1e4 * rvir / dd,
z * 1e4 * rvir / dd,
rstride=10,
cstride=10,
color='0.5',
alpha=0.7)
#plot the random points
ax4.scatter(rd['x'] * 2e4,
rd['y'] * 2e4,
rd['z'] * 2e4,
color='b',
s=30,
alpha=0.8)
ax1.set_xlabel(r'Projected Separation [$10^{-4}$ deg]')
ax2.set_xlabel(r'Projected Distance [kpc]')
ax3.set_xlabel(r'$\Delta$RA [$10^{-4}$ deg]')
#ax4.set_xlabel(r'$\Delta$RA [$10^{-4}$ deg]')
ax3.set_ylabel(r'$\Delta$DEC [$10^{-4}$ deg]')
#ax4.set_ylabel(r'$\Delta$DEC [$10^{-4}$ deg]')
ax3.set_xlim(-200, 200)
ax3.set_ylim(-100, 100)
ax4.set_xlim3d(-150, 150)
ax4.set_ylim3d(-150, 150)
ax4.set_zlim3d(-150, 150)
plt.annotate(text, (0.5, 0.95),
xycoords='figure fraction',
ha='center',
va='center',
fontsize=12)
plt.savefig('RandomHalo.pdf')
def quickPlot(galaxies=170, rvir=0.105):
"""
A single halo test
:param galaxies: number of subhalo galaxies
:param rvir: radius of the main halo [Mpc]
"""
conversion = 0.000277777778 # degree to arcsecond
#redshift
z = 4.477
#get the angular diameter distance to the galaxy
dd = cosmocalc(z, 71.0, 0.28)['PS_kpc'] #in kpc / arc seconds
#position of the main galaxy
RADeg1 = 189.41432
decDeg1 = 62.166702
# get the random values
rds = rand.randomUnitSphere(galaxies)
# convert them to Cartesian coord
rd = conv.convertSphericalToCartesian(rvir/dd/2., rds['theta'], rds['phi'])
data = {'CD': np.matrix('-1 0; 0 1'), 'RA': RADeg1, 'DEC': decDeg1, 'X': rd['y'], 'Y': rd['z']}
result = conv.RAandDECfromStandardCoordinates(data)
RADeg2 = result['RA']
decDeg2 = result['DEC']
#calculate the angular separation on the sky
sep = Coords.calcAngSepDeg(RADeg1, decDeg1, RADeg2, decDeg2) / 2.
physical_distance = sep * dd / conversion
text = r'Random Positions, z={0:.2f}, radius of the halo = {1:.0f} kpc'.format(z, 1e3 * rvir)
#make a figure
fig = plt.figure(figsize=(12,12))
ax1 = fig.add_subplot(221)
ax2 = fig.add_subplot(222)
ax3 = fig.add_subplot(223)
rect = fig.add_subplot(224, visible=False).get_position()
ax4 = Axes3D(fig, rect)
ax1.hist(sep * 1e4, bins=12)
ax2.hist(physical_distance, bins=12)
ax3.scatter(0, 0, c='k', s=120, marker='s')
ax3.scatter((RADeg2 - RADeg1) * 1e4, (decDeg2 - decDeg1) * 1e4, c='b', s=50, marker='o', alpha=0.45)
#generate a sphere
u = np.linspace(0, 2 * np.pi, 100)
v = np.linspace(0, np.pi, 100)
x = np.outer(np.cos(u), np.sin(v))
y = np.outer(np.sin(u), np.sin(v))
z = np.outer(np.ones(np.size(u)), np.cos(v))
#plot it as a wire frame
ax4.plot_wireframe(x*1e4*rvir/dd, y*1e4*rvir/dd, z*1e4*rvir/dd,
rstride=10, cstride=10, color='0.5', alpha=0.7)
#plot the random points
ax4.scatter(rd['x']*2e4, rd['y']*2e4, rd['z']*2e4, color='b', s=30, alpha=0.8)
ax1.set_xlabel(r'Projected Separation [$10^{-4}$ deg]')
ax2.set_xlabel(r'Projected Distance [kpc]')
ax3.set_xlabel(r'$\Delta$RA [$10^{-4}$ deg]')
#ax4.set_xlabel(r'$\Delta$RA [$10^{-4}$ deg]')
ax3.set_ylabel(r'$\Delta$DEC [$10^{-4}$ deg]')
#ax4.set_ylabel(r'$\Delta$DEC [$10^{-4}$ deg]')
ax3.set_xlim(-200, 200)
ax3.set_ylim(-100, 100)
ax4.set_xlim3d(-150, 150)
ax4.set_ylim3d(-150, 150)
ax4.set_zlim3d(-150, 150)
plt.annotate(text, (0.5, 0.95), xycoords='figure fraction', ha='center', va='center', fontsize=12)
plt.savefig('RandomHalo.pdf')