def coadd_coords(lon,lat,labels=None,index=None):
"""
Calculated the median coordinates of each object.
Returns
out : Dictionary of average coordinates.
"""
if labels is None:
labels = np.ones(len(lon),dtype=int)
if index is None: index = np.unique(labels)
x,y,z = healpy.rotator.dir2vec(lon,lat,lonlat=True)
# Mean coordinates
#x_out = nd.mean(x,labels=labels,index=index)
#y_out = nd.mean(y,labels=labels,index=index)
#z_out = nd.mean(z,labels=labels,index=index)
# Median coordinates
x_out = nd.median(x,labels=labels,index=index)
y_out = nd.median(y,labels=labels,index=index)
z_out = nd.median(z,labels=labels,index=index)
# std coordinates
#x_std = nd.standard_deviation(x,labels=labels,index=index)
#y_std = nd.standard_deviation(y,labels=labels,index=index)
#z_std = nd.standard_deviation(z,labels=labels,index=index)
lon_out, lat_out = healpy.rotator.vec2dir(x_out,y_out,z_out,lonlat=True)
# What about adding a dispersion?
return lon_out % 360.,lat_out
def test_median02():
a = np.array([[1, 2, 0, 1],
[5, 3, 0, 4],
[0, 0, 0, 7],
[9, 3, 0, 0]])
output = ndimage.median(a)
assert_almost_equal(output, 1.0)
def test_median02():
a = np.array([[1, 2, 0, 1],
[5, 3, 0, 4],
[0, 0, 0, 7],
[9, 3, 0, 0]])
output = ndimage.median(a)
assert_almost_equal(output, 1.0)
def procesar_imagen(archivo="", factor=0.5, f="white"):
'''
A partir de archivo procesa una imagen del Sol,
detectando los centros de masa con un factor
por encima del valor medio y por debajo del maximo.
Crea un archivo procesado en la carpeta output/img/~.png
y otro en output/mosaico_~.png con una
superpocision de todos los valores.
@param archivo: la ruta del archivo a procesar
@param factor: Desde 0 para valor promedio, hasta 1 para valor maximo
@param f: puede ser 'white' para usar white_~.png o 'sun' para usar sun_~.png
'''
# Lee la imagen del disco duro
img = misc.imread("img/" + archivo, 1)
# Calcula el valor maximo de intensidad para un pixel e invierte todos los valores, de modo que ahora oscuridad = max y luz = 0
maximum = ndimage.maximum(img)
img = maximum - img
# calcula para los nuevos datos maximo y valor promedio
median = ndimage.median(img)
maximum = ndimage.maximum(img)
# Crea filtro con un criterio muy relativo :P
filtro = img > median + (maximum - median) * factor
img = img * filtro
# Calcula los centros de masa con un maximo de NUM posibles manchas
lbl, num = ndimage.label(img)
# centros de masa
center_of_mass = ndimage.measurements.center_of_mass(
img, lbl, range(2, num + 1))
# crear mapa de imagen con elementos calculados
arch = archivo.split("_", 3)
sun = misc.imread("lib/img/" + f + "_" + arch[2] + ".png", 1)
mosaico = misc.imread("output/mosaico_" + arch[2] + ".png", 1)
for elemento in center_of_mass:
y = int(elemento[0])
x = int(elemento[1])
mosaico[y, x] = 0
sun[y, x] = 0
# configura un nuevo nombre para la imagen en output/
archivo = arch[0] + "_" + arch[1] + "_" + arch[2] + ".png"
misc.imsave("output/img/" + archivo, sun)
misc.imsave("output/mosaico_" + arch[2] + ".png", mosaico)
def depth(infile,nside=NSIDE,signal_to_noise=10.):
MAGS = bfields('MAG_PSF',BANDS)
MAGERRS = bfields('MAGERR_PSF',BANDS)
SPREADS = bfields('WAVG_SPREAD_MODEL',BANDS)
# From propagation of errors:
# mag = -2.5 * log10(flux)
# magerr = -2.5/ln(10) * fluxerr/flux
mag_snr = (2.5/np.log(10)) / (signal_to_noise)
logger.info(infile)
ret = dict()
for band,mag,magerr,spread in zip(BANDS,MAGS,MAGERRS,SPREADS):
data = fitsio.read(infile,columns=['RA','DEC',mag,magerr,spread])
h, edges = np.histogram(data[mag], bins=np.arange(17, 30, 0.1))
mag_bright_end = edges[np.argmax(h)] - 3.
cut = (np.fabs(data[spread]) < 0.002) & (data[mag] > mag_bright_end) & (data[mag] < 30.)
d = data[cut]
if len(d) < 2:
logger.warn("Insufficent objects in %s-band"%band)
ret[band] = [np.array([],dtype=int),np.array([])]
continue
pix = ang2pix(nside,d['RA'],d['DEC'])
match = ugali.utils.projector.match(d['RA'],d['DEC'],d['RA'],d['DEC'],nnearest=2)
delta_mag = d[mag][match[1]] - d[mag][match[0]]
delta_log_magerr = np.log10(d[magerr][match[1]]) - np.log10(d[magerr][match[0]])
old = np.seterr(divide='ignore',invalid='ignore')
ratio = delta_log_magerr / delta_mag
cut_nan_inf = np.isfinite(ratio) & (delta_mag > 0.5)
np.seterr(**old)
if cut_nan_inf.sum() < 2:
logger.warn("Insufficent objects in %s-band"%band)
ret[band] = [np.array([],dtype=int),np.array([])]
continue
kde = scipy.stats.gaussian_kde(ratio[cut_nan_inf])
values = np.linspace(0., 1., 1000)
kde_values = kde.evaluate(values)
slope = values[np.argmax(kde_values)]
maglims = d[mag] - ((np.log10(d[magerr]) - np.log10(mag_snr)) / slope)
hpx = np.unique(pix)
maglim = nd.median(maglims,labels=pix,index=hpx)
ret[band] = [hpx,maglim]
return ret
def test_median01():
a = np.array([[1, 2, 0, 1],
[5, 3, 0, 4],
[0, 0, 0, 7],
[9, 3, 0, 0]])
labels = np.array([[1, 1, 0, 2],
[1, 1, 0, 2],
[0, 0, 0, 2],
[3, 3, 0, 0]])
output = ndimage.median(a, labels=labels, index=[1, 2, 3])
assert_array_almost_equal(output, [2.5, 4.0, 6.0])
def test_median03():
a = np.array([[1, 2, 0, 1],
[5, 3, 0, 4],
[0, 0, 0, 7],
[9, 3, 0, 0]])
labels = np.array([[1, 1, 0, 2],
[1, 1, 0, 2],
[0, 0, 0, 2],
[3, 3, 0, 0]])
output = ndimage.median(a, labels=labels)
assert_almost_equal(output, 3.0)
def test_median01():
a = np.array([[1, 2, 0, 1],
[5, 3, 0, 4],
[0, 0, 0, 7],
[9, 3, 0, 0]])
labels = np.array([[1, 1, 0, 2],
[1, 1, 0, 2],
[0, 0, 0, 2],
[3, 3, 0, 0]])
output = ndimage.median(a, labels=labels, index=[1, 2, 3])
assert_array_almost_equal(output, [2.5, 4.0, 6.0])
def test_median03():
a = np.array([[1, 2, 0, 1],
[5, 3, 0, 4],
[0, 0, 0, 7],
[9, 3, 0, 0]])
labels = np.array([[1, 1, 0, 2],
[1, 1, 0, 2],
[0, 0, 0, 2],
[3, 3, 0, 0]])
output = ndimage.median(a, labels=labels)
assert_almost_equal(output, 3.0)
def centroid(lon, lat, stat='median', labels=None, index=None):
if labels is None:
labels = np.ones(len(lon), dtype=int)
if index is None: index = np.unique(labels)
x, y, z = hp.rotator.dir2vec(lon, lat, lonlat=True)
if stat == 'mean':
x_out = nd.mean(x, labels=labels, index=index)
y_out = nd.mean(y, labels=labels, index=index)
z_out = nd.mean(z, labels=labels, index=index)
elif stat == 'median':
x_out = nd.median(x, labels=labels, index=index)
y_out = nd.median(y, labels=labels, index=index)
z_out = nd.median(z, labels=labels, index=index)
else:
msg = "Unrecognized stat: %s" % stat
raise Exception(msg)
lon_out, lat_out = hp.rotator.vec2dir(x_out, y_out, z_out, lonlat=True)
return lon_out % 360., lat_out
def draw_magerr(mag,magerr,**kwargs):
kwargs.setdefault('fmt','o')
bins = kwargs.pop('bins',np.linspace(16,25,100))
ax = plt.gca()
labels = np.digitize(mag,bins)
index = np.unique(labels)
centers = ((bins[1:]+bins[:-1])/2.)[index]
median = nd.median(magerr,labels=labels,index=index)
mean = nd.mean(magerr,labels=labels,index=index)
std = nd.standard_deviation(magerr,labels=labels,index=index)
ax.errorbar(centers,mean,yerr=std,**kwargs)
return median,mean,std
def center_estimation_Tsukada2021b(img, x1st, y1st, radius=60):
### triming
img = np.pad(img, radius, mode="constant", constant_values=np.nan)
x1st, y1st = x1st + radius, y1st + radius
img = SAR.lee_filter(img[y1st-radius:y1st+radius, x1st-radius:x1st+radius], 3)
### eye extraction
eye_region_candidates = img < 0.8 * np.nanmean(img) # high_wind_region = img > 1.2 * np.nanmean(img)
lbls, nlbls = ndi.label(eye_region_candidates)
sizes = np.bincount(lbls.ravel())[1:]
img = np.where(np.isnan(img), 0, img)
eyewall_features = {}
for nlbl in range(1, nlbls+1):
if sizes[nlbl-1] < size_min or sizes[nlbl-1] > size_max:
continue
lbl = (lbls == nlbl)
dilated = ndi.binary_dilation(lbl, structure=np.ones([3,3]), iterations=15)
eyewall = dilated ^ lbl
eyewall_feature = ndi.median(img, eyewall) - ndi.minimum(img, lbl)
if ~np.isnan(eyewall_feature):
eyewall_features[nlbl] = eyewall_feature
eye_lbl = max(eyewall_features, key=eyewall_features.get)
y, x = ndi.center_of_mass(lbls == eye_lbl)
x, y = x + x1st - 2 * radius, y + y1st - 2 * radius
# rmax = int(rmax/(110*np.abs(sar.lat[0]-sar.lat[1])))
# flags = cv2.INTER_LINEAR + cv2.WARP_FILL_OUTLIERS + cv2.WARP_POLAR_LINEAR
# wind_norm = cv2.normalize(img, None, 0, 255, cv2.NORM_MINMAX, dtype=cv2.CV_8U)
# polar = cv2.warpPolar(wind_norm, (rmax, int(360/0.5)), (x, y), rmax, flags)
# diff = np.diff(polar.astype(np.float), axis=1)
# blurred_diff = ndi.gaussian_filter(diff, sigma=5)
# eye_extents = np.argmax(blurred_diff, axis=1) # align with 0.5 deg
# eye_region_polar = np.zeros_like(polar)
# for i, eye_r in enumerate(eye_extents):
# eye_region_polar[i, :eye_r] = 1
# eye_region = SAR.get_cart(eye_region_polar, rmax, rmax * 2, rmax * 2)
# plt.imshow(eye_region)
# plt.show()
# ydiff, xdiff = ndi.measurements.center_of_mass(eye_region) - np.array([rmax, rmax])
# x, y = x + xdiff, y + ydiff
return x, y
def removeObjects(binarized):
"""
This function identify objects as contigunious spot of white on a
black image and remove the bigger and smaller ones.
---------------------------
@args:
binarized: 2D array
Represent the inversed color binarized picture from which
objects bigger than the mean will be taken from.
@return:
binarized: 2D array
"cleaned" array
scale: double
Represent the size of the mean object
---------------------------
This function improve line detection and "clean" the image.
A coefficient could be added to the function to change the value of
height and width of the deleted object, or to choose only horizontal
or vertical objects.
"""
labels, n = ocrolib.morph.label(binarized)
objects = ocrolib.morph.find_objects(labels)
# Calculate the median of all objects
bysize = sorted(objects, key=ocrolib.sl.area)
scalemap = np.zeros(binarized.shape)
for sub_object in bysize:
if np.amax(scalemap[sub_object]) > 0:
continue
scalemap[sub_object] = ocrolib.sl.area(sub_object)**0.5
scale = ndi.median(scalemap[(scalemap > 3) & (scalemap < 100)])
# Take the mesurement of small objects
sums = ndi.measurements.sum(binarized, labels, range(n + 1))
sums = sums[labels]
# Find all objects and remove big ones
for i, b in enumerate(objects):
if (ocrolib.sl.width(b) > scale * 8) \
or (ocrolib.sl.height(b) > scale * 8):
labels[b][labels[b] == i + 1] = 0
# Recreate an array without big objects
binarized = np.array(labels != 0, 'B')
# Remove small objects from this image
binarized = np.minimum(binarized, 1 - (sums > 0) * (sums < scale))
return binarized, scale
def getCMFromImage(archivo="", factor=0.5):
'''
A partir de archivo procesa una imagen del Sol,
detectando los centros de masa con un factor
por encima del valor medio y por debajo del maximo.
@param archivo: la ruta del archivo a procesar
@param factor: Desde 0 para valor promedio, hasta 1 para valor maximo
@return: retorna centros de masa para la imagen
'''
# Lee la imagen del disco duro
img = misc.imread("img/" + archivo, 1)
# Calcula el valor maximo de intensidad para un pixel e invierte todos los valores, de modo que ahora oscuridad = max y luz = 0
maximum = ndimage.maximum(img)
img = maximum - img
# calcula para los nuevos datos maximo y valor promedio
median = ndimage.median(img)
maximum = ndimage.maximum(img)
# Crea filtro con un criterio muy relativo :P
filtro = img > median + (maximum - median) * factor
img = img * filtro
# Calcula los centros de masa con un maximo de NUM posibles manchas
lbl, num = ndimage.label(img)
# centros de masa
# center_of_mass = ndimage.measurements.center_of_mass(img, lbl, range(2, num + 1))
# El primer valor es el centro del Sol
center_of_mass = ndimage.measurements.center_of_mass(
img, lbl, range(1, num + 1))
# lee el tiempo de toma de la imagen
arch = archivo.split("_", 2)
tiempo = time.strptime(arch[0] + arch[1], "%Y%m%d%H%M%S")
# return centros de masa y tiempo de foto
return center_of_mass, time.mktime(tiempo)
def rms_photometry(catfile,nside=64,band=None,plot=False):
""" Calculate photometric repeatability """
if not os.path.exists(catfile):
msg = "Couldn't find %s"%catfile
raise Exception(msg)
columns = ['RA','DEC']
spread,nepochs = bfields(['WAVG_SPREAD_MODEL','NEPOCHS'],band)
mag,magerr,magrms = bfields(['WAVG_MAG_PSF','WAVG_MAGERR_PSF','WAVG_MAGRMS_PSF'],band)
columns += [spread, nepochs, mag, magerr, magrms]
# Hack to get pixel location
hpx = int(catfile.split('_')[-1].split('.')[0])
#hpx = ang2pix(NSIDE, cat['RA'], cat['DEC'])
ra,dec = pix2ang(NSIDE, hpx)
msg = '%s (RA,DEC) = %.2f,%.2f'%(os.path.basename(catfile),ra,dec)
print(msg)
#print "Getting coadd catalog: DES"
cat = load_infiles([catfile],columns)
# Select stars with 16 < r < 20 and 0.0 < (g-i) < 1.5
sel = (np.fabs(cat[spread]) < 0.002) & \
(cat[mag] > 16) & (cat[mag] < 18) &\
(cat[magrms] < 90) &\
(cat[nepochs] > 1)
cat = cat[sel]
if len(cat) == 0:
msg = "WARNING: No objects passing selection in: %s"%catfile
print(msg)
return np.array([],dtype=int), np.array([])
pix = ang2pix(nside,cat['RA'],cat['DEC'])
upix = np.unique(pix)
stat = nd.median(cat[magrms],labels=pix,index=upix)
if False:
plt.figure()
plt.hist(cat[magrms],bins=50)
import pdb; pdb.set_trace()
return upix,stat
def teff(infile,nside=NSIDE,mode='median'):
TEFFS = bfields('TEFF',BANDS)
logger.info(infile)
ret = dict()
for band,teff in zip(BANDS,TEFFS):
data = fitsio.read(infile,columns=['RA','DEC',teff])
pix = ang2pix(nside,data['RA'],data['DEC'])
hpx = np.unique(pix)
if mode.lower() is 'median':
teff_value = nd.median(data[teff],labels=pix,index=hpx)
elif mode.lower() is 'mean':
teff_value = nd.mean(data[teff],labels=pix,index=hpx)
else:
msg = 'Unrecognized mode: %s'%mode
raise Exception(msg)
ret[band] = [hpx,maglim]
return ret
def get_spectrum(dark_noise, channels, mask):
"""Get spectrum from image set using mask.
The median of the masked area is extracted, the camera dark noise
subtracted, and normalized.
Parameters
----------
dark_noise : int
Intrinsic dark noise of camera. Image taken when shutter closed.
channels : slice, list
Slice of channels
mask : NumPy array
Labeled mask.
"""
data = np.array([ndi.median(ch, mask) for ch in channels])
data -= dark_noise # Dark noise subtract
data /= data.sum() # Normalize to 1.
return data
def imagestats():
import math
import scipy.ndimage as nd
image = getvar('image', inmodule=True)
if image is None:
print('could not find image')
return
do = getvar('do', inmodule=True)
if do is None:
print('could not find display object')
return
data = image.data[:, :, do.zp].squeeze()
dmed = nd.median(data)
print("mean:\t\t%f" % data.mean())
print("variance:\t%f" % data.var())
print("std dev:\t%f" % data.std())
print("median:\t\t%f" % dmed)
print("med-sqrt:\t%f" % math.sqrt(dmed))
def distance(args,plot=False):
nice = os.nice(0)
os.nice(10-nice)
pix,nside = args
catfile = glob.glob('cat/*_%05d.fits'%pix)[0]
if not os.path.exists(catfile):
msg = "Couldn't find %s"%catfile
raise Exception(msg)
print(catfile)
columns = [OBJECT_ID,'RA','DEC']
spread,mag,nepochs = bfields(['WAVG_SPREAD_MODEL','MAG_PSF','NEPOCHS'],band)
columns += [spread,mag,nepochs]
cat = load_infiles([catfile],columns)
sel = (np.fabs(cat[spread])<0.002)&(cat[mag]>16)&(cat[mag]<22)&(cat[nepochs] > 1)
cat = cat[sel]
if len(cat) == 0:
print("WARNING: No catalog objects passing selection")
return np.array([],dtype=int), np.array([])
ra,dec = cat['RA'],cat['DEC']
m = ugali.utils.projector.match(ra,dec,ra,dec,nnearest=2)
sep = m[-1]
hpx = ang2pix(nside,ra,dec)
peak = nd.median(sep,labels=hpx,index=np.unique(hpx))
if plot:
plt.figure()
draw_angsep(sep)
if isinstance(plot,basestring):
outfile = plot
plt.savefig(outfile,bbox_inches='tight')
return hpx,peak
def test_stat_funcs_2d():
a = np.array([[5,6,0,0,0], [8,9,0,0,0], [0,0,0,3,5]])
lbl = np.array([[1,1,0,0,0], [1,1,0,0,0], [0,0,0,2,2]])
mean = ndimage.mean(a, labels=lbl, index=[1, 2])
assert_array_equal(mean, [7.0, 4.0])
var = ndimage.variance(a, labels=lbl, index=[1, 2])
assert_array_equal(var, [2.5, 1.0])
std = ndimage.standard_deviation(a, labels=lbl, index=[1, 2])
assert_array_almost_equal(std, np.sqrt([2.5, 1.0]))
med = ndimage.median(a, labels=lbl, index=[1, 2])
assert_array_equal(med, [7.0, 4.0])
min = ndimage.minimum(a, labels=lbl, index=[1, 2])
assert_array_equal(min, [5, 3])
max = ndimage.maximum(a, labels=lbl, index=[1, 2])
assert_array_equal(max, [9, 5])
def test_stat_funcs_2d():
a = np.array([[5,6,0,0,0], [8,9,0,0,0], [0,0,0,3,5]])
lbl = np.array([[1,1,0,0,0], [1,1,0,0,0], [0,0,0,2,2]])
mean = ndimage.mean(a, labels=lbl, index=[1, 2])
assert_array_equal(mean, [7.0, 4.0])
var = ndimage.variance(a, labels=lbl, index=[1, 2])
assert_array_equal(var, [2.5, 1.0])
std = ndimage.standard_deviation(a, labels=lbl, index=[1, 2])
assert_array_almost_equal(std, np.sqrt([2.5, 1.0]))
med = ndimage.median(a, labels=lbl, index=[1, 2])
assert_array_equal(med, [7.0, 4.0])
min = ndimage.minimum(a, labels=lbl, index=[1, 2])
assert_array_equal(min, [5, 3])
max = ndimage.maximum(a, labels=lbl, index=[1, 2])
assert_array_equal(max, [9, 5])
img_v2 = cv2.imread(r'data/e-codices_kba-0016-2_006v_large.jpg') img_v2 = cv2.cvtColor(img_v2, cv2.COLOR_BGR2RGB).astype(np.uint16) gray2 = cv2.cvtColor(img_v2, cv2.COLOR_BGR2GRAY) img_mean = np.mean(gray2) #todo Exercise 8.1: computing projection profiles img_contrast = cv2.normalize(gray2, gray2, 0, 255, cv2.NORM_MINMAX) img_contrast = img_contrast - img_mean imagetoshow2D(img_contrast) projection = np.sum(img_contrast, axis=1) #todo Exercise 8.2: searching for line starts subImg = img_contrast[:, :int(img_contrast.shape[1] * 0.25)] temp = np.sum(subImg, axis=1) cost = median(temp, size=5) cost[cost > 0] = 0 x = abs(cost) peaks, _ = find_peaks(x, height=0) idx = np.argmin(x) plt.figure() plt.plot(x) plt.plot(peaks, x[peaks], "x") plt.show() #todo Exercise 8.3: compute seams img_cost = ComputerEnergy(img_contrast) m = np.max(np.sum(img_cost, axis=0)) img_cost[:, 0] = m img_cost[peaks, 0] = -m allpath = seam_carving_row(img_cost, peaks) for path in allpath:
def external_astrometry(catfile,catalog='GAIA',nside=64,band='r',plot=False):
#nice = os.nice(0)
#os.nice(10-nice)
if band=='all': band = 'r'
if not os.path.exists(catfile):
msg = "Couldn't find %s"%catfile
raise Exception(msg)
columns = [OBJECT_ID,'RA','DEC']
spread,mag,nepochs = bfields(['WAVG_SPREAD_MODEL','MAG_PSF','NEPOCHS'],band)
columns += [spread,mag,nepochs]
# Hack to get pixel location
hpx = int(catfile.split('_')[-1].split('.')[0])
#hpx = ang2pix(NSIDE, cat['RA'], cat['DEC'])
ra,dec = pix2ang(NSIDE, hpx)
radius = np.degrees(healpy.max_pixrad(NSIDE))
msg = '%s (RA,DEC,RAD) = %.2f,%.2f,%.2f'%(os.path.basename(catfile),ra,dec,radius)
print(msg)
#print "Getting coadd catalog: DES"
cat = load_infiles([catfile],columns)
# Select stars with 16 < mag < 21
sel = (np.fabs(cat[spread])<0.002) & \
(cat[mag]>16) & \
(cat[mag]<21) & \
(cat[nepochs] > 1)
cat = cat[sel]
if len(cat) == 0:
msg = "WARNING: No objects passing selection in: %s"%catfile
print(msg)
return np.array([],dtype=int), np.array([])
#print "Getting external catalog: %s"%catalog
if catalog in list(CATALOGS.keys()):
ext = get_vizier_catalog(ra,dec,radius,**CATALOGS[catalog])
else:
ext = get_local_catalog(ra,dec,radius,catalog)
m = match_query(cat['RA'],cat['DEC'],ext['_RAJ2000'],ext['_DEJ2000'])
# Use a fairly wide matching radius (2 arcsec)
cut = 2.0
sel = m[-1]*3600. < cut
sepdeg = m[-1][sel]
sepsec = m[-1][sel] * 3600.
sepmas = sepsec * 1000.
sep = sepmas
pix = ang2pix(nside,cat['RA'][sel],cat['DEC'][sel])
upix = np.unique(pix)
try:
peak = nd.median(sep,labels=pix,index=upix)
except ValueError:
import pdb; pdb.set_trace()
if plot:
plt.figure()
draw_angsep(sep,bins=np.linspace(0,cut*1000.,101))
if isinstance(plot,basestring):
outfile = plot
plt.savefig(outfile,bbox_inches='tight')
#return cat,ext,m
return upix,peak
def internal_astrometry(catfile,hpxfile,nside=128,band='r',plot=False):
""" Calculate internal relative astrometry.
Parameters
----------
catfile : merged catalog file
hpxfile : single epoch catalog file(s)
nside : nside for calculation
band : band to use
plot : plot output
Returns
-------
stats : output statistics
"""
nice = os.nice(0)
os.nice(10-nice)
if band=='all': band = 'r'
#print catfile,hpxfile,nside
#catfile = glob.glob('cat/*_%05d.fits'%pix)[0]
if not os.path.exists(catfile):
msg = "Couldn't find %s"%catfile
raise Exception(msg)
columns = [OBJECT_ID,'RA','DEC']
spread,mag,nepochs = bfields(['WAVG_SPREAD_MODEL','MAG_PSF','NEPOCHS'],band)
columns += [spread,mag,nepochs]
cat = load_infiles([catfile],columns)
# Select stars with 17 < mag < 21
sel = (np.fabs(cat[spread])<0.002) & \
(cat[mag]>17) & \
(cat[mag]<21) & \
(cat[nepochs] > 1)
cat = cat[sel]
if len(cat) == 0:
print("WARNING: No objects passing selection in: %s"%catfile)
return np.array([],dtype=int), np.array([])
#hpxfiles = glob.glob('hpx/%s/*_%05d.fits'%(band,pix))
hpx = load_infiles(hpxfile, [OBJECT_ID, 'RA', 'DEC'])
hpx = hpx[np.in1d(hpx[OBJECT_ID],cat[OBJECT_ID])]
if len(hpx) == 0:
print("WARNING: No matched objects in: %s"%hpxfile)
return np.array([],dtype=int), np.array([])
#keyfile = 'key/key_hpx_%05d.fits'%pix
#key = load_infiles([keyfile],[OBJECT_ID,'FILENAME','OBJECT_NUMBER'])
#key = key[np.in1d(key[OBJECT_ID],cat[OBJECT_ID])]
#
#key_id = np.char.add(key['FILENAME'],key['OBJECT_NUMBER'].astype(str))
#hpx_id = np.char.add(hpx['FILENAME'],hpx['OBJECT_NUMBER'].astype(str))
#
#hpx = hpx[np.in1d(hpx_id,key_id)]
uid,inv,cts = np.unique(hpx[OBJECT_ID],False,True,True)
# Make sure that the order matches between coadd and single epoch.
if not np.all(uid == cat[OBJECT_ID]):
cat = cat[np.in1d(cat[OBJECT_ID],hpx[OBJECT_ID])]
if not np.all(uid == cat[OBJECT_ID]):
cat = cat[np.argsort(cat[OBJECT_ID])]
assert np.all(uid == cat[OBJECT_ID])
ra,dec = cat['RA'][inv],cat['DEC'][inv]
sepdeg = angsep(ra,dec,hpx['RA'],hpx['DEC'])
sepsec = sepdeg * 3600.
sepmas = sepsec * 1000.
sel = [sepsec > 1e-5]
sep = sepmas[sel]
pix = ang2pix(nside,ra[sel],dec[sel])
upix = np.unique(pix)
peak = nd.median(sep,labels=pix,index=upix)
if plot:
plt.figure()
draw_angsep(sep)
if isinstance(plot,basestring):
outfile = plot
plt.savefig(outfile,bbox_inches='tight')
return upix,peak
def gaia_photometry(catfile,nside=64,band=None,plot=False,version='edr3'):
if not os.path.exists(catfile):
msg = "Couldn't find %s"%catfile
raise Exception(msg)
#columns = [OBJECT_ID,'RA','DEC']
columns = ['RA','DEC']
spread,nepochs = ['WAVG_SPREAD_MODEL_R','NEPOCHS_R']
mag_g,mag_r,mag_i,mag_z = bfields(['MAG_PSF'],['g','r','i','z'])
#mag_g,mag_r,mag_i,mag_z = bfields(['WAVG_MAG_PSF'],['g','r','i','z'])
columns += [spread, nepochs, mag_g, mag_r, mag_i, mag_z]
# Hack to get pixel location
hpx = int(catfile.split('_')[-1].split('.')[0])
#hpx = ang2pix(NSIDE, cat['RA'], cat['DEC'])
ra,dec = pix2ang(NSIDE, hpx)
radius = np.degrees(hp.max_pixrad(NSIDE))
msg = '%s (RA,DEC,RAD) = %.2f,%.2f,%.2f'%(os.path.basename(catfile),ra,dec,radius)
print(msg)
#print "Getting coadd catalog: DES"
cat = load_infiles([catfile],columns)
# Select stars with 16 < r < 20 and 0.0 < (g-i) < 1.5
sel = (np.fabs(cat[spread])<0.002) & \
(cat[mag_g]<90) & (cat[mag_r]<90) & (cat[mag_i]<90) & (cat[mag_z]<90) & \
(cat[mag_r]>16) & (cat[mag_r]<20) & \
((cat[mag_g] - cat[mag_i]) > 0.0) & \
((cat[mag_g] - cat[mag_i]) < 1.5) & \
(cat[nepochs] > 1)
cat = cat[sel]
if len(cat) == 0:
msg = "WARNING: No objects passing selection in: %s"%catfile
print(msg)
return np.array([],dtype=int), np.array([])
#msg = "Getting external catalog: %s"%catalog
ext = get_gaia_catalog(hpx,version=version)
m = match_query(cat['RA'],cat['DEC'],ext['RA'],ext['DEC'])
# Use a fairly wide matching radius (2 arcsec)
cut = 1.0
sel = m[-1]*3600. < cut
cat_match = cat[m[0][sel]]
ext_match = ext[m[1][sel]]
cat_G = gaia_transform(cat_match[mag_g],cat_match[mag_r],cat_match[mag_i],cat_match[mag_z],
version=version)
# Need to put Gaia flux onto the AB system
ext_G = -2.5 * np.log10(ext_match['PHOT_G_MEAN_FLUX']) + 25.7934
diff = cat_G - ext_G
pix = ang2pix(nside,cat_match['RA'],cat_match['DEC'])
upix = np.unique(pix)
stat = nd.median(diff,labels=pix,index=upix)
if False:
plt.figure()
plt.hist(cat_G - ext_G)
import pdb; pdb.set_trace()
return upix,stat
def test_median_gh12836_bool():
# test boolean addition fix on example from gh-12836
a = np.asarray([1, 1], dtype=bool)
output = ndimage.median(a, labels=np.ones((2, )), index=[1])
assert_array_almost_equal(output, [1.0])
#plt.errorbar(bincenters, y,
#yerr = np.sqrt(y), fmt='r.', markersize='5', alpha = 0.7, elinewidth= 2 )
#plt.plot(bincenters[y!=0], y[y!=0], 'r', alpha = 0.8, lw = 2, label = strLabel)
errorfill(bincenters[y != 0], y[y != 0], np.sqrt(y[y != 0]), color='navy')
#plt.xlim(binEdges[1:][0], xlim2)
#plt.ylim(1, )
#plt.yscale('log')
#if (strLabel != '#Grids in Halo'): plt.xscale('log')
#plt.xlabel(strLabel)
#plt.ylabel("pdf")
#plt.legend(loc = "upper right")
minnstrEachBlob = np.array(ndi.median(nstream, labels=labels3d, index=label1d))
array1d = minnstrEachBlob
strLabel = r'median($n_{str}$)'
xlim1 = np.min(array1d)
xlim2 = np.max(array1d)
bins = np.logspace(np.log10(xlim1), np.log10(xlim2), nbins)
y, binEdges = np.histogram(array1d, bins=bins, density=False)
bincenters = 0.5 * (binEdges[1:] + binEdges[:-1])
#plt.errorbar(bincenters, y,
#yerr = np.sqrt(y), fmt='b.', markersize='5', alpha = 0.7, elinewidth= 2 )
#
#plt.plot(bincenters[y!=0], y[y!=0], 'b', alpha = 0.8, lw = 2, label = strLabel)
plt.xlabel(r'${\rm ZP_{GCM} - ZP_{qSLR}}')
plt.ylabel("Number of CCDs")
gcm_bands = dict()
qslr_bands = dict()
diff_bands = dict()
for b in BANDS:
zp_g = gcm['MAG_ZERO'][gcm['BAND'] == b]
zp_q = qslr['MAG_ZERO'][qslr['BAND'] == b]
delta = zp_g - zp_q
labels = qslr[HPX][qslr['BAND'] == b]
index = np.unique(labels)
for data, out in [(zp_g, gcm_bands), (zp_q, qslr_bands),
(delta, diff_bands)]:
skymap = blank(nside)
skymap[index] = nd.median(data, labels=labels, index=index)
out[b] = skymap
diff_colors = dict()
for name, (b1, b2) in COLORS:
gcm_color = gcm_bands[b1] - gcm_bands[b2]
qslr_color = qslr_bands[b1] - qslr_bands[b2]
skymap = gcm_color - qslr_color
diff_colors[name] = skymap
for b in BANDS:
skymap = diff_bands[b]
plt.figure()
im = plotting.draw_footprint(skymap)
plt.colorbar(im, label=r'${\rm ZP_{GCM} - ZP_{qSLR}}$')
plt.title('Median Zeropoint Offset (%s-band)' % (b))
#plt.errorbar(bincenters, y,
#yerr = np.sqrt(y), fmt='r.', markersize='5', alpha = 0.7, elinewidth= 2 )
#plt.plot(bincenters[y!=0], y[y!=0], 'r', alpha = 0.8, lw = 2, label = strLabel)
errorfill(bincenters[y != 0], y[y != 0], np.sqrt(y[y != 0]), color='navy')
#plt.xlim(binEdges[1:][0], xlim2)
#plt.ylim(1, )
#plt.yscale('log')
#if (strLabel != '#Grids in Halo'): plt.xscale('log')
#plt.xlabel(strLabel)
#plt.ylabel("pdf")
#plt.legend(loc = "upper right")
minffEachBlob = np.array(ndi.median(flip, labels=labels3d, index=label1d))
array1d = minffEachBlob
strLabel = r'median($n_{str}$)'
xlim1 = np.min(array1d)
xlim2 = np.max(array1d)
bins = np.logspace(np.log10(xlim1), np.log10(xlim2), nbins)
y, binEdges = np.histogram(array1d, bins=bins, density=False)
bincenters = 0.5 * (binEdges[1:] + binEdges[:-1])
#plt.errorbar(bincenters, y,
#yerr = np.sqrt(y), fmt='b.', markersize='5', alpha = 0.7, elinewidth= 2 )
#
#plt.plot(bincenters[y!=0], y[y!=0], 'b', alpha = 0.8, lw = 2, label = strLabel)
for i,v in enumerate(['WAVG_MAG_PSF','MAG_PSF']):
for c,b in zip(['g','r','gold','indigo','gray'],BANDS):
var = v+'_%s'%b.upper()
alpha = 0.5 if i > 0 else 1.0
plt.hist(coadd[var],alpha=alpha,label=var,color=c,**kwargs)
plt.legend(loc='upper left',fontsize=10)
plt.savefig(pltdir+'mag_%05d.png'%opts.pix,bbox_inches='tight')
for c,b in zip(['g','r','gold','indigo','gray'],BANDS):
plt.figure()
for i,v in enumerate(['WAVG_MAG_PSF','MAG_PSF']):
var = v+'_%s'%b.upper()
err = var.replace('MAG_','MAGERR_')
bins = kwargs['bins']
labels = np.digitize(coadd[var],bins)
index = np.unique(labels)
medians = nd.median(coadd[err],labels=labels,index=index)
alpha = 0.5 if i > 0 else 1.0
mag = bins[index[:-1]]; magerr = medians[:-1]
plt.plot(mag,magerr,'-o',c=c,alpha=alpha,label=var)
argmax = np.argmax(magerr > 0.1)
maglim = (mag[argmax] + mag[argmax-1])/2.
if i==1: plot_peak(maglim,'%.1f'%maglim,color=c,lw=1.5,ls='--')
plt.axhline(0.1,color='gray',lw=1.5,ls='--')
plt.xlim(16,25); plt.ylim(0,0.5)
plt.xlabel('MAG_PSF')
plt.xlabel('MAGERR_PSF')
plt.legend(loc='upper left',fontsize=10)
plt.savefig(pltdir+'magerr_%s_%05d.png'%(b,opts.pix),bbox_inches='tight')
float
) # for some reason ndi functions over int images give weird results
result = {}
for mask_name in MASKS:
values = MASKS[mask_name](img, brain_mask, head_mask)
for metric_name in METRICS:
column = mask_name + ' ' + metric_name
result[column] = METRICS[metric_name](values)
result['histogram bg peak'] = mri_bg_peak(
img) # no reason to use it with masks
return result
METRICS = {
'mean': lambda x: ndi.mean(x),
'median': lambda x: ndi.median(x),
'std': lambda x: ndi.standard_deviation(x),
}
MASKS = {
'brain': lambda i, b, h: i[b],
'head': lambda i, b, h: i[h],
'animal': lambda i, b, h: i[b | h],
'image': lambda i, b, h: i.flatten(),
'back': lambda i, b, h: i[~(b | h)],
}
def mri_bg_peak(img: np.ndarray) -> float:
"""
This function fits a gaussian onto image intensity histogram peak (the one, corresponding to background).
def test_median_no_int_overflow():
# test integer overflow fix on example from gh-12836
a = np.asarray([65, 70], dtype=np.int8)
output = ndimage.median(a, labels=np.ones((2, )), index=[1])
assert_array_almost_equal(output, [67.5])
#!/bin/env python
import sys
from scipy import stats
from scipy import ndimage
import numpy
import pysam
numbers = []
samfile = pysam.AlignmentFile(sys.argv[1], 'rb', check_sq=False)
for read in samfile:
numbers.append(int(read.query_length))
describe = stats.describe(numbers)
print "\t".join(["nobs", "min", "max", "mean", "std", "median", "coverage"])
out = [
describe.nobs, describe.minmax[0], describe.minmax[1], describe.mean,
ndimage.standard_deviation(numpy.array(numbers)),
ndimage.median(numbers),
ndimage.sum(numpy.array(numbers)) / float(3100000000)
]
print "\t".join(map(str, out))
def median_in_circle(imagen, r, centro=None):
labels = ring_mask(imagen.shape[0], 0, r, center=centro)
return nd.median(imagen, labels, 1)
