def match_cat(tdata_1,tdata_2,radius,data1cat,data2cat):
print('Matching catalogs')
maxmatch = 1
NSIDE = 4096
if data1cat == 'y3gold':
ra_1 = tdata_1['ALPHAWIN_J2000']
dec_1 = tdata_1['DELTAWIN_J2000']
elif data1cat == 'y1gold':
ra_1 = tdata_1['RA']
dec_1 = tdata_1['DEC']
elif data1cat == 'deep':
ra_1 = tdata_1['ra']
dec_1 = tdata_1['dec']
else:
ra_1 = tdata_1['RA']
ra_1 = tdata_1['DEC']
if data2cat == 'deep':
ra_2 = tdata_2['ra']
dec_2 = tdata_2['dec']
elif data2cat == 'hsc':
ra_2 = tdata_2['ra']
dec_2 = tdata_2['dec']
else:
ra_1 = tdata_1['ra']
ra_1 = tdata_1['dec']
matches = smatch.match(ra_1, dec_1, radius, ra_2, dec_2, nside=NSIDE, maxmatch=maxmatch)
return matches
def get_matches(ra1, dec1, ra2, dec2, mpc_deg=None, scale='physical',
rmin=0.1, rmax=20.0, nside=64):
"""Cross-match coordinates.
Parameters
----------
ra1, dec1 : float or numpy array
Coordinates of the first group of objects.
ra2, dec2 : float or numpy array
Coordinates of the second group of objects.
rmin : float, optional
Minimum radius limit. Default: 0.1 Mpc.
rmax : float, optional
Maximum radius limit. Default: 20.0 Mpc.
mpc_deg : float
Phyiscal size in unit of Mpc per degree. Default: None.
scale : string
Coordinate type, `physical` or `angular`. Default: `physical`.
nside : int
Default: 64
Return
------
Boolen array for matched objects.
"""
if scale not in ["physical", "angular"]:
raise Exception("Scale should be either physical or angular")
max_radius, min_radius = rmax, rmin
if scale == 'physical':
if mpc_deg is None:
raise Exception("Please provide correct redshift information")
# Physical match: max_radius is in Mpc.
max_degrees = (max_radius / mpc_deg)
else:
# Angular match: max_radius is already in unit of degree
max_degrees = max_radius
# TODO: larger nside makes the matching process slower,
matches = smatch.match(ra1, dec1, max_degrees, ra2, dec2,
nside=nside, maxmatch=0)
# this is a bit dirty... But I don't think it is terrible.
matches.dtype.names = ('i1', 'i2', 'dist')
matches['dist'] = np.rad2deg(np.arccos(matches['dist']))
# Convert the distances from angular unit into physical unit if necessary.
if scale == 'physical':
if isinstance(mpc_deg, np.ndarray):
matches['dist'] *= mpc_deg[matches['i1']]
else:
matches['dist'] *= mpc_deg
# Find the matched ones between min_radius and max_radius
mask = (matches['dist'] > min_radius) & (matches['dist'] <= max_radius)
return matches[mask]
def smatch_catalog(table1,
table2,
rmatch,
index='index',
ra1='ra',
dec1='dec',
ra2='ra',
dec2='dec',
nside=4096,
maxmatch=1,
join_type='left',
filled=True,
verbose=True):
"""Match two catalogs using smatch."""
# Perform the match using smatch
matches = smatch.match(table1[ra1],
table1[dec1],
rmatch,
table2[ra2],
table2[dec2],
nside=nside,
maxmatch=maxmatch)
if verbose:
print("# Find %d matches !" % len(matches))
if len(matches) > 1:
# Add an index column
cat2 = copy.deepcopy(table2)
cat2.add_column(
Column(data=np.full(len(table2), -999, dtype=np.int64),
name=index))
cat2.add_column(Column(data=np.full(len(table2), 0.0), name='cosdist'))
cat2[index][matches['i2']] = table1[index][matches['i1']].data
cat2['cosdist'][matches['i2']] = matches['cosdist']
# Join two tables
table_output = join(table1, cat2, keys=index, join_type=join_type)
# Only keep the unique
table_output.sort('cosdist')
table_output.reverse()
table_output = unique(table_output,
keys=index,
silent=True,
keep='first')
table_output.remove_column('cosdist')
if filled:
return table_output.filled()
return table_output
return None
def Match(self, ObjectCatalog):
"""
If we have more than one catalog and we want to know how many points their have in common we will use this method.
Firts of all, what we do here is a condition, if the catalogs have been read properly and if the catalogs are not the same we will start making the matching.
To make the match we import from smatch library the function match, where it needs the RA and DEC of each catalog.
Finally, when the match have been completed, we add the matching values to an array for each catalog.
Parameter
---------
ObjectCatalog: `int`
Here, we write the other catalog with which we are going to make the matching.
Returns
-------
None.
"""
import smatch
if self.Lleno and self.nombre != ObjectCatalog.nombre:
self.ra1_matched = self.datos[self.RA]
self.dec1_matched = self.datos[self.DEC]
self.datos2 = ObjectCatalog.datos
self.RA2 = ObjectCatalog.RA
self.DEC2 = ObjectCatalog.DEC
self.ra2_matched = self.datos2[self.RA2]
self.dec2_matched = self.datos2[self.DEC2]
#print('La longitud de', self.RA, 'es', len(ra1_matched), 'y la de', self.DEC, 'es', len(dec1_matched) )
#print('La longitud de', ObjectCatalog[2], 'es', len(ra2_matched), 'y la de', ObjectCatalog[3], 'es', len(dec2_matched), 'donde estos datos pertenecen al catalogo', ObjectCatalog[0])
self.matches = smatch.match(self.ra1_matched,
self.dec1_matched,
self.radius,
self.ra2_matched,
self.dec2_matched,
nside=self.nside,
maxmatch=self.maxmatch)
self.assoc1 = self.datos[self.matches['i1']]
self.assoc2 = self.datos2[self.matches['i2']]
print(
'La longitud del catalogo ya habiendo realizado el matched es',
len(self.assoc1))
self.match = True
self.nombreMatch = ObjectCatalog.nombre
self.MainCatalog()
else:
print('No se ha podido hacer el match')
magnitud_error_sharks = dat.field('APERMAG3ERR')
ra_2mass = dat2.field('RAJ2000')
dec_2mass = dat2.field('DEJ2000')
magnitud_2mass = dat2.field('Kmag')
magnitud_error_2mass = dat2.field('e_Kmag')
nside = 4096 # healpix nside
maxmatch = 1 # return closest match
radius = 1 / 3600.
# ra,dec,radius in degrees
matches = smatch.match(ra_sharks,
dec_sharks,
radius,
ra_2mass,
dec_2mass,
nside=nside,
maxmatch=maxmatch)
#print (matches)
ra_sharks_matched = ra_sharks[matches['i1']]
dec_sharks_matched = dec_sharks[matches['i1']]
magnitud_sharks_matched = magnitud_sharks[matches['i1']]
magnitud_error_sharks_matched = magnitud_error_sharks[matches['i1']]
ra_2mass_matched = ra_2mass[matches['i2']]
dec_2mass_matched = dec_2mass[matches['i2']]
magnitud_2mass_matched = magnitud_2mass[matches['i2']]
magnitud_error_2mass_matched = magnitud_error_2mass[matches['i2']]
mag1 = dat.field('MAG_AUTO')
magger1 = dat.field('MAGERR_AUTO')
ra2 = dat2.field('RAJ2000')
dec2 = dat2.field('DEJ2000')
kmag2 = dat2.field('Kmag')
ekmag2 = dat2.field('e_Kmag')
nside = 4096 # healpix nside
maxmatch = 1 # return closest match
radius = 1 / 3600.
# ra,dec,radius in degrees
matches = smatch.match(ra1,
dec1,
radius,
ra2,
dec2,
nside=nside,
maxmatch=maxmatch)
#print (matches)
ra1matched = ra1[matches['i1']]
dec1matched = dec1[matches['i1']]
mag1matched = mag1[matches['i1']]
magger1matched = magger1[matches['i1']]
ra2matched = ra2[matches['i2']]
dec2matched = dec2[matches['i2']]
kmag2matched = kmag2[matches['i2']]
ekmag2matched = ekmag2[matches['i2']]
cosgamma = []
def makeReferenceMatches(self, refLoader):
"""
Make an absolute reference match catalog.
Parameters
----------
refLoader: `object`
Object which has refLoader.getFgcmReferenceStarsHealpix
"""
# can we use the better smatch code?
try:
import smatch
hasSmatch = True
except ImportError:
hasSmatch = False
ipring = hp.ang2pix(self.starConfig['coarseNSide'],
np.radians(90.0 - self.objIndexCat['dec']),
np.radians(self.objIndexCat['ra']))
hpix, revpix = esutil.stat.histogram(ipring, rev=True)
pixelCats = []
nBands = len(self.starConfig['referenceFilterNames'])
dtype = [('fgcm_id', 'i4'), ('refMag', 'f4', nBands),
('refMagErr', 'f4', nBands)]
gdpix, = np.where(hpix > 0)
for ii, gpix in enumerate(gdpix):
p1a = revpix[revpix[gpix]:revpix[gpix + 1]]
# Choose the center of the stars...
raWrap = self.objIndexCat['ra'][p1a]
if (raWrap.min() < 10.0) and (raWrap.max() > 350.0):
hi, = np.where(raWrap > 180.0)
raWrap[hi] -= 360.0
meanRA = np.mean(raWrap)
if meanRA < 0.0:
meanRA += 360.0
else:
meanRA = np.mean(raWrap)
meanDec = np.mean(self.objIndexCat['dec'][p1a])
dist = esutil.coords.sphdist(meanRA, meanDec,
self.objIndexCat['ra'][p1a],
self.objIndexCat['dec'][p1a])
rad = dist.max()
# Note nside2resol returns radians of the pixel along a side...
if rad < np.degrees(
hp.nside2resol(self.starConfig['coarseNSide']) / 2.):
# If it's a smaller radius, read the circle
refCat = refLoader.getFgcmReferenceStarsSkyCircle(
meanRA, meanDec, rad,
self.starConfig['referenceFilterNames'])
else:
# Otherwise, this will always work
refCat = refLoader.getFgcmReferenceStarsHealpix(
self.starConfig['coarseNSide'], ipring[p1a[0]],
self.starConfig['referenceFilterNames'])
if refCat.size == 0:
# No stars in this pixel. That's okay.
continue
if hasSmatch:
matches = smatch.match(self.objIndexCat['ra'][p1a],
self.objIndexCat['dec'][p1a],
self.starConfig['matchRadius'] / 3600.0,
refCat['ra'],
refCat['dec'],
nside=self.starConfig['matchNSide'],
maxmatch=1)
i1 = matches['i1']
i2 = matches['i2']
else:
htm = esutil.htm.HTM(11)
matcher = esutil.htm.Matcher(11, self.objIndexCat['ra'][p1a],
self.objIndexCat['dec'][p1a])
matches = matcher.match(refCat['ra'],
refCat['dec'],
self.starConfig['matchRadius'] /
3600.0,
maxmatch=1)
# matches[0] -> m1 -> array from matcher.match() call (refCat)
# matches[1] -> m2 -> array from htm.Matcher() (self.objIndexCat)
i2 = matches[0]
i1 = matches[1]
# i1 -> objIndexCat[p1a]
# i2 -> refCat
if i1.size == 0:
# No matched stars in this pixel. That's okay.
continue
pixelCat = np.zeros(i1.size, dtype=dtype)
pixelCat['fgcm_id'] = self.objIndexCat['fgcm_id'][p1a[i1]]
pixelCat['refMag'][:, :] = refCat['refMag'][i2, :]
pixelCat['refMagErr'][:, :] = refCat['refMagErr'][i2, :]
pixelCats.append(pixelCat)
self.fgcmLog.info(
"Found %d reference matches in pixel %d (%d of %d)." %
(pixelCat.size, ipring[p1a[0]], ii, gdpix.size - 1))
# Now assemble
count = 0
for pixelCat in pixelCats:
count += pixelCat.size
self.referenceCat = np.zeros(count, dtype=dtype)
ctr = 0
for pixelCat in pixelCats:
self.referenceCat[ctr:ctr + pixelCat.size] = pixelCat
ctr += pixelCat.size
# and clear memory
pixelCat = None
def makeMatchedStars(self, raArray, decArray, filterNameArray):
"""
Make matched stars, from pre-loaded arrays. Requires self.objCat was
generated from makePrimaryStars().
parameters
----------
raArray: double array
RA for each observation
decArray: double array
Dec for each observation
filterNameArray: numpy string array
filterName for each array
"""
if (self.objCat is None):
raise ValueError("Must run makePrimaryStars first")
# can we use the better smatch code?
try:
import smatch
hasSmatch = True
except ImportError:
hasSmatch = False
if (raArray.size != decArray.size
or raArray.size != filterNameArray.size):
raise ValueError(
"raArray, decArray, filterNameArray must be same length")
# translate filterNameArray to bandArray ... can this be made faster, or
# does it need to be?
filterNameArrayIsEncoded = False
try:
test = filterNameArray[0].decode('utf-8')
filterNameArrayIsEncoded = True
except AttributeError:
pass
bandArray = np.zeros_like(filterNameArray)
for filterName in self.filterNames:
if filterNameArrayIsEncoded:
use, = np.where(filterNameArray == filterName.encode('utf-8'))
else:
use, = np.where(filterNameArray == filterName)
bandArray[use] = self.starConfig['filterToBand'][filterName]
self.fgcmLog.info("Matching positions to observations...")
if (hasSmatch):
# faster smatch...
matches = smatch.match(self.objCat['ra'],
self.objCat['dec'],
self.starConfig['matchRadius'] / 3600.0,
raArray,
decArray,
nside=self.starConfig['matchNSide'],
maxmatch=0)
i1 = matches['i1']
i2 = matches['i2']
else:
# slower htm matching...
htm = esutil.htm.HTM(11)
matcher = esutil.htm.Matcher(11, self.objCat['ra'],
self.objCat['dec'])
matches = matcher.match(raArray,
decArray,
self.starConfig['matchRadius'] / 3600.,
maxmatch=0)
# matches[0] -> m1 -> array from matcher.match() call (ra/decArray)
# matches[1] -> m2 -> array from htm.Matcher() (self.objCat)
i2 = matches[0]
i1 = matches[1]
self.fgcmLog.info("Collating observations")
nObsPerObj, obsInd = esutil.stat.histogram(i1, rev=True)
if (nObsPerObj.size != self.objCat.size):
raise ValueError(
"Number of primary stars (%d) does not match observations (%d)."
% (self.objCat.size, nObsPerObj.size))
# and our simple classifier
# 1 is a good star, 0 is bad.
objClass = np.zeros(self.objCat.size, dtype='i2')
# We may have no "required" bands beyond being in one of the primary bands
if len(self.starConfig['requiredBands']) > 0:
# which stars have at least minPerBand observations in each required band?
reqBands = np.array(self.starConfig['requiredBands'],
dtype=bandArray.dtype)
# this could be made more efficient
self.fgcmLog.info("Computing number of observations per band")
nObs = np.zeros((reqBands.size, self.objCat.size), dtype='i4')
for i in range(reqBands.size):
use, = np.where(bandArray[i2] == reqBands[i])
hist = esutil.stat.histogram(i1[use],
min=0,
max=self.objCat.size - 1)
nObs[i, :] = hist
# cut the star list to those with enough per band
minObs = nObs.min(axis=0)
# make sure we have enough per band
gd, = np.where(minObs >= self.starConfig['minPerBand'])
objClass[gd] = 1
else:
objClass[:] = 1
gd, = np.where(objClass == 1)
self.fgcmLog.info(
"There are %d stars with at least %d observations in each required band."
% (gd.size, self.starConfig['minPerBand']))
# cut the density of stars down with sampling.
theta = (90.0 - self.objCat['dec'][gd]) * np.pi / 180.
phi = self.objCat['ra'][gd] * np.pi / 180.
ipring = hp.ang2pix(self.starConfig['densNSide'], theta, phi)
hist, rev = esutil.stat.histogram(ipring, rev=True)
high, = np.where(hist > self.starConfig['densMaxPerPixel'])
ok, = np.where(hist > 0)
self.fgcmLog.info("There are %d/%d pixels with high stellar density" %
(high.size, ok.size))
for i in range(high.size):
i1a = rev[rev[high[i]]:rev[high[i] + 1]]
cut = np.random.choice(i1a,
size=i1a.size -
self.starConfig['densMaxPerPixel'],
replace=False)
objClass[gd[cut]] = 0
# redo the good object selection after sampling
gd, = np.where(objClass == 1)
dtype = [('fgcm_id', 'i4'), ('ra', 'f8'), ('dec', 'f8'),
('obsarrindex', 'i4'), ('nobs', 'i4')]
hasExtraQuantities = False
if len(self.starConfig['quantitiesToAverage']) > 0:
hasExtraQuantities = True
for quant in self.starConfig['quantitiesToAverage']:
dtype.extend([(quant, 'f4')])
# create the object catalog index
self.objIndexCat = np.zeros(gd.size, dtype=dtype)
self.objIndexCat['fgcm_id'][:] = self.objCat['fgcm_id'][gd]
self.objIndexCat['ra'][:] = self.objCat['ra'][gd]
self.objIndexCat['dec'][:] = self.objCat['dec'][gd]
# this is the number of observations per object
self.objIndexCat['nobs'][:] = nObsPerObj[gd]
# and the index is given by the cumulative sum
self.objIndexCat['obsarrindex'][1:] = np.cumsum(nObsPerObj[gd])[:-1]
# Copy in the extra quantities
if hasExtraQuantities:
for quant in self.starConfig['quantitiesToAverage']:
self.objIndexCat[quant][:] = self.objCat[quant][gd]
# and we need to create the observation indices from the obsarrindex
nTotObs = self.objIndexCat['obsarrindex'][-1] + self.objIndexCat[
'nobs'][-1]
self.obsIndexCat = np.zeros(nTotObs, dtype=[('obsindex', 'i4')])
ctr = 0
self.fgcmLog.info("Spooling out %d observation indices." % (nTotObs))
for i in gd:
self.obsIndexCat[ctr:ctr +
nObsPerObj[i]] = i2[obsInd[obsInd[i]:obsInd[i +
1]]]
ctr += nObsPerObj[i]
def makePrimaryStars(self,
raArray,
decArray,
filterNameArray,
extraQuantityArrays=None,
bandSelected=False,
brightStarRA=None,
brightStarDec=None,
brightStarRadius=None):
"""
Make primary stars, from pre-loaded arrays
parameters
----------
raArray: double array
RA for each observation
decArray: double array
Dec for each observation
filterNameArray: string array
Array of filterNames.
extraQuantityArrays: numpy recarray, optional
Record array of extra quantities to average. Default None.
bandSelected: bool, default=False
Has the input raArray/decArray been pre-selected by band?
brightStarRA: double array, optional
RA for bright stars for mask
brightStarDec: double array, optional
Dec for bright stars for mask
brightStarRadius: float array, optional
Radius for bright stars for mask
Output attributes
-----------------
objCat: numpy recarray
Catalog of unique objects selected from primary band
"""
# can we use the better smatch code?
try:
import smatch
hasSmatch = True
self.fgcmLog.info("Using smatch for matching.")
except ImportError:
hasSmatch = False
self.fgcmLog.info("Using htm for matching.")
if (raArray.size != decArray.size):
raise ValueError("raArray, decArray must be same length.")
if (raArray.size != filterNameArray.size):
raise ValueError("raArray, filterNameArray must be same length.")
# Prepare bright stars if necessary...
if (brightStarRA is not None and brightStarDec is not None
and brightStarRadius is not None):
if (brightStarRA.size != brightStarDec.size
or brightStarRA.size != brightStarRadius.size):
raise ValueError(
"brightStarRA/Dec/Radius must have same length")
cutBrightStars = True
else:
cutBrightStars = False
# Define the dtype
dtype = [('fgcm_id', 'i4'), ('ra', 'f8'), ('dec', 'f8')]
hasExtraQuantities = False
if len(self.starConfig['quantitiesToAverage']) > 0:
if extraQuantityArrays is None:
raise RuntimeError(
"Cannot set quantitiesToAverage without passing extraQuantityArrays"
)
hasExtraQuantities = True
for quant in self.starConfig['quantitiesToAverage']:
dtype.extend([(quant, 'f4')])
if quant not in extraQuantityArrays.dtype.names:
raise RuntimeError(
"quantity to average %s not in extraQuantityArrays" %
(quant))
pixelCats = []
# Split into pixels
ipring = hp.ang2pix(self.starConfig['coarseNSide'],
(90.0 - decArray) * np.pi / 180.,
raArray * np.pi / 180.)
hpix, revpix = esutil.stat.histogram(ipring, rev=True)
gdpix, = np.where(hpix > 0)
self.fgcmLog.info("Matching primary stars in %d pixels" % (gdpix.size))
for ii, gpix in enumerate(gdpix):
# This is the array of all the observations in the coarse pixel
p1a = revpix[revpix[gpix]:revpix[gpix + 1]]
if p1a.size == 0:
continue
bandPixelCat = None
filterNameArrayIsEncoded = False
try:
test = filterNameArray[0].decode('utf-8')
filterNameArrayIsEncoded = True
except AttributeError:
pass
# loop over bands...
for primaryBand in self.starConfig['primaryBands']:
# We first need to select based on the band, not on the filter name
useFlag = None
for filterName in self.filterNames:
if (self.starConfig['filterToBand'][filterName] ==
primaryBand):
if useFlag is None:
if filterNameArrayIsEncoded:
useFlag = (filterNameArray[p1a] ==
filterName.encode('utf-8'))
else:
useFlag = (filterNameArray[p1a] == filterName)
else:
if filterNameArrayIsEncoded:
useFlag |= (filterNameArray[p1a] ==
filterName.encode('utf-8'))
else:
useFlag = (filterNameArray[p1a] == filterName)
raArrayUse = raArray[p1a[useFlag]]
decArrayUse = decArray[p1a[useFlag]]
if hasExtraQuantities:
extraQuantityArraysUse = extraQuantityArrays[p1a[useFlag]]
if raArrayUse.size == 0:
self.fgcmLog.info("Nothing found for pixel %d" %
(ipring[p1a[0]]))
continue
esutil.numpy_util.to_native(raArrayUse, inplace=True)
esutil.numpy_util.to_native(decArrayUse, inplace=True)
if hasSmatch:
# faster match...
self.fgcmLog.info("Starting smatch...")
matches = smatch.match(raArrayUse,
decArrayUse,
self.starConfig['matchRadius'] /
3600.0,
raArrayUse,
decArrayUse,
nside=self.starConfig['matchNSide'],
maxmatch=0)
i1 = matches['i1']
i2 = matches['i2']
self.fgcmLog.info("Finished smatch.")
else:
# slower htm matching...
htm = esutil.htm.HTM(11)
matcher = esutil.htm.Matcher(11, raArrayUse, decArrayUse)
matches = matcher.match(raArrayUse,
decArrayUse,
self.starConfig['matchRadius'] /
3600.0,
maxmatch=0)
i1 = matches[1]
i2 = matches[0]
"""
# Try this instead...
counter = np.zeros(raArrayUse.size, dtype=np.int64)
minId = np.zeros(raArrayUse.size, dtype=np.int64) + raArrayUse.size + 1
raMeanAll = np.zeros(raArrayUse.size, dtype=np.float64)
decMeanAll = np.zeros(raArrayUse.size, dtype=np.float64)
# Count the number of observations of each matched observation
np.add.at(counter, i1, 1)
# Find the minimum id of the match to key on a unique value
# for each
np.fmin.at(minId, i2, i1)
# Compute the mean ra/dec
np.add.at(raMeanAll, i1, raArrayUse[i2])
raMeanAll /= counter
np.add.at(decMeanAll, i1, decArrayUse[i2])
decMeanAll /= counter
uId = np.unique(minId)
bandPixelCatTemp = np.zeros(uId.size, dtype=dtype)
bandPixelCatTemp['ra'] = raMeanAll[uId]
bandPixelCatTemp['dec'] = decMeanAll[uId]
# Any extra quantities?
if len(self.starConfig['quantitiesToAverage']) > 0:
for quant in enumerate(self.starConfig['quantitiesToAverage']):
quantMeanAll = np.zeros(raArrayUse.size, dtype=np.float64)
np.add.at(quantMeanAll, i1, extraQuantityArrays[quant][p1a[useFlag[i2]]])
quantMeanAll /= counter
bandPixelCatTemp[quant] = quantMeanAll[uId]
"""
# This is the official working version, but slower
fakeId = np.arange(p1a.size)
hist, rev = esutil.stat.histogram(fakeId[i1], rev=True)
if (hist.max() == 1):
self.fgcmLog.warn(
"No matches found for pixel %d, band %s!" %
(ipring[p1a[0]], primaryBand))
continue
maxObs = hist.max()
# how many unique objects do we have?
histTemp = hist.copy()
count = 0
for j in range(histTemp.size):
jj = fakeId[j]
if (histTemp[jj] >= self.starConfig['minPerBand']):
i1a = rev[rev[jj]:rev[jj + 1]]
histTemp[i2[i1a]] = 0
count = count + 1
# make a temporary catalog...
bandPixelCatTemp = np.zeros(count, dtype=dtype)
# Rotate. This works for DES, but maybe not in general?
raTemp = raArrayUse.copy()
hi, = np.where(raTemp > 180.0)
raTemp[hi] -= 360.0
# Compute mean ra/dec
index = 0
for j in range(hist.size):
jj = fakeId[j]
if (hist[jj] >= self.starConfig['minPerBand']):
i1a = rev[rev[jj]:rev[jj + 1]]
starInd = i2[i1a]
# make sure this doesn't get used again
hist[starInd] = 0
bandPixelCatTemp['ra'][index] = np.sum(
raTemp[starInd]) / starInd.size
bandPixelCatTemp['dec'][index] = np.sum(
decArrayUse[starInd]) / starInd.size
if hasExtraQuantities:
for quant in self.starConfig[
'quantitiesToAverage']:
ok, = np.where(
(extraQuantityArraysUse[quant +
'_err'][starInd] >
0.0))
wt = 1. / extraQuantityArraysUse[
quant + '_err'][starInd[ok]]**2.
bandPixelCatTemp[quant][index] = np.sum(
wt * extraQuantityArraysUse[quant][
starInd[ok]]) / np.sum(wt)
index = index + 1
# Restore negative RAs
lo, = np.where(bandPixelCatTemp['ra'] < 0.0)
bandPixelCatTemp['ra'][lo] += 360.0
# Match to previously pixel catalog if available, and remove dupes
if bandPixelCat is None:
# First time through, these are all new objects
bandPixelCat = bandPixelCatTemp
self.fgcmLog.info(" Found %d primary stars in %s band" %
(bandPixelCatTemp.size, primaryBand))
else:
# We already have objects, need to match/append
if hasSmatch:
bandMatches = smatch.match(
bandPixelCat['ra'],
bandPixelCat['dec'],
self.starConfig['matchRadius'] / 3600.0,
bandPixelCatTemp['ra'],
bandPixelCatTemp['dec'],
maxmatch=0)
i1b = matches['i1']
i2b = matches['i2']
else:
matcher = esutil.htm.Matcher(11, bandPixelCat['ra'],
bandPixelCat['dec'])
matches = matcher.match(
bandPixelCatTemp['ra'],
bandPixelCatTemp['dec'],
self.starConfig['matchRadius'] / 3600.0,
maxmatch=0)
i1b = matches[1]
i2b = matches[0]
# Remove all matches from the temp catalog
bandPixelCatTemp = np.delete(bandPixelCatTemp, i2b)
self.fgcmLog.info(
" Found %d new primary stars in %s band" %
(bandPixelCatTemp.size, primaryBand))
bandPixelCat = np.append(bandPixelCat, bandPixelCatTemp)
if bandPixelCat is not None:
# Append to list of catalogs...
pixelCats.append(bandPixelCat)
self.fgcmLog.info(
"Found %d unique objects in pixel %d (%d of %d)." %
(bandPixelCat.size, ipring[p1a[0]], ii, gdpix.size - 1))
# now assemble into a total objCat
count = 0
for pixelCat in pixelCats:
count += pixelCat.size
self.objCat = np.zeros(count, dtype=dtype)
ctr = 0
for pixelCat in pixelCats:
self.objCat[ctr:ctr + pixelCat.size] = pixelCat
ctr += pixelCat.size
# and clear memory
pixelCat = None
self.objCat['fgcm_id'] = np.arange(count) + 1
self.fgcmLog.info("Found %d unique objects with >= %d observations." %
(count, self.starConfig['minPerBand']))
if (cutBrightStars):
self.fgcmLog.info("Matching to bright stars for masking...")
if (hasSmatch):
# faster smatch...
matches = smatch.match(brightStarRA,
brightStarDec,
brightStarRadius,
self.objCat['ra'],
self.objCat['dec'],
nside=self.starConfig['matchNSide'],
maxmatch=0)
i1 = matches['i1']
i2 = matches['i2']
else:
# slower htm matching...
htm = esutil.htm.HTM(11)
matcher = esutil.htm.Matcher(10, brightStarRA, brightStarDec)
matches = matcher.match(self.objCat['ra'],
self.objCat['dec'],
brightStarRadius,
maxmatch=0)
# matches[0] -> m1 -> array from matcher.match() call (self.objCat)
# matches[1] -> m2 -> array from htm.Matcher() (brightStar)
i1 = matches[1]
i2 = matches[0]
self.fgcmLog.info("Cutting %d objects too near bright stars." %
(i2.size))
self.objCat = np.delete(self.objCat, i2)
# and remove stars with near neighbors
self.fgcmLog.info("Matching stars to neighbors...")
if (hasSmatch):
# faster smatch...
matches = smatch.match(self.objCat['ra'],
self.objCat['dec'],
self.starConfig['isolationRadius'] / 3600.0,
self.objCat['ra'],
self.objCat['dec'],
nside=self.starConfig['matchNSide'],
maxmatch=0)
i1 = matches['i1']
i2 = matches['i2']
else:
# slower htm matching...
htm = esutil.htm.HTM(11)
matcher = esutil.htm.Matcher(11, self.objCat['ra'],
self.objCat['dec'])
matches = matcher.match(self.objCat['ra'],
self.objCat['dec'],
self.starConfig['isolationRadius'] /
3600.0,
maxmatch=0)
i1 = matches[1]
i2 = matches[0]
use, = np.where(i1 != i2)
if (use.size > 0):
neighbored = np.unique(i2[use])
self.fgcmLog.info(
"Cutting %d objects within %.2f arcsec of a neighbor" %
(neighbored.size, self.starConfig['isolationRadius']))
self.objCat = np.delete(self.objCat, neighbored)
