def findMulti(self, M, threshold=None, num_iter=None):
if threshold is None:
threshold = self.threshold
if num_iter is None:
num_iter = self.num_iterations
for trans in range(2):
for i in range(num_iter):
new_streak = self.findSingle(M,
transpose=trans,
threshold=threshold)
if empty(new_streak):
break
else:
new_streak.subtractStreak(M)
if self.use_subtract_mean: M -= np.nanmean(M)
# np.nan_to_num(M, copy=False)
if self.use_show:
new_streak.plotLines()
f = plt.gcf()
f.canvas.draw()
f.canvas.flush_events()
return M
def getRadonVariance(self, transpose=0):
""" Get the partial Radon transforms of the background noise for some transpose.
Lazy Reloading: only delete old var-maps if input size changed (or
if we changed expansion mode) and then calculate the var-maps on demand.
"""
# check if we need to recalculate the var map
if not empty(self._size_var) and (
not compare_size(self.im_size, self._size_var)
or self.useExpand() != self._expanded_var):
if self.debug_bit: print("Clearing the Radon var-maps")
self._radon_var_map = [
] # clear this to be lazy loaded with the right size
self._radon_var_map_trans = []
# if there is no var map, we need to lazy load it
if empty(self._radon_var_map):
# do we have a variance map or scalar??
# if empty(self._input_var):
# self._input_var = self._default_var_scalar
#
# if scalar(self._input_var):
# self._var_scalar = self._input_var
# self._var_map = self._input_var * np.ones(self.im_size, dtype='float32')
# else:
# self._var_scalar = np.median(self.input_var)
# self._var_map = self._input_var
self._size_var = self.im_size
self._expanded_var = self.useExpand()
self._radon_var_map = FRT(self.var_map,
partial=True,
expand=self._expanded_var,
transpose=False)
self._radon_var_map_trans = FRT(self.var_map,
partial=True,
expand=self._expanded_var,
transpose=True)
if transpose:
return self._radon_var_map_trans
else:
return self._radon_var_map
def write_to_disk(self, filename=None):
if empty(filename):
if empty(self.filename):
return
else:
filename = os.path.splitext(self.filename)[0]+'.h5'
with h5py.File(filename, 'a') as hf:
numbers=[int(re.search(r'\d+$', k).group()) for k in hf.keys()]
if empty(numbers):
new_number = 1
else:
new_number = max(numbers) + 1
ds = hf.create_dataset('streak_%03d' % (new_number), data=self.image_cutout)
ds.attrs['corner'] = self.corner_cutout
def makeCutout(self, size=None):
if empty(self.image):
return;
if empty(size):
size = self.cut_size;
d1 = int(np.floor(size/2))
d2 = int(np.ceil(size/2))
x1 = int(round(self.mid_x_full))-d1
x2 = int(round(self.mid_x_full))+d2
y1 = int(round(self.mid_y_full))-d1
y2 = int(round(self.mid_y_full))+d2
self.image_cutout = self.image[y1:y2,x1:x2]
self.size_cutout = (size, size)
self.corner_cutout = (y1, x1)
def input_var(self, val):
# if working with scalar variance, only need to update the scaling of var maps
if scalar(val) and scalar(self._input_var):
if not empty(self._radon_var_map):
self._radon_var_map = [
m * val / self.input_var for m in self._radon_var_map
]
if not empty(self._radon_var_map_trans):
self._radon_var_map_trans = [
m * val / self.input_var for m in self._radon_var_map_trans
]
else: # new or old input_var is not scalar, must recalculate var maps
self.clearVarMap()
self._input_var = val
if scalar(val):
self._var_scalar = val
self._var_map = None
else:
self._var_scalar = np.median(val)
self._var_map = val
def update_from_finder(self, finder):
if empty(finder): return
for att in self.__dict__.keys(): # load all attributes in "self" that exist in "finder"
if hasattr(finder, att) and not callable(getattr(finder,att)):
setattr(self, att, getattr(finder, att))
self.noise_var = finder.var_scalar
self.psf_sigma = finder.sigma_psf
self.corner_section = finder.current_section_corner
self.was_convolved = bool(finder.use_conv)
self.was_expanded = bool(finder.useExpand())
def __set_input_psf(self, val):
if scalar(val):
if empty(self.input_psf) or not scalar(
self.input_psf) or val != self.input_psf:
self._psf = gaussian2D(val)
self._sigma_psf = val
elif isinstance(val, np.ndarray) and val.ndim == 2:
if scalar(self.input_psf
) or val.shape != self._input_psf.shape or np.any(
val != self.input_psf):
self._psf = val
a = fit_gaussian(
val) # run a short minimization 2D fitter to gaussian
self._sigma_psf = (a.x[1] +
a.x[2]) / 2 # average the x and y sigma
else:
raise ValueError("input_psf must be a scalar or a numpy array")
self._input_psf = val
self._psf = self._psf / np.sqrt(np.sum(self._psf**2)) # normalized PSF
def sigma_psf(self):
if empty(self._input_psf):
self.input_psf = self._default_sigma_psf # go through the setter for input_psf and calculate psf and sigma_psf
return self._sigma_psf
def var_map(self):
if empty(self._var_map) and not empty(self.im_size):
return self.var_scalar * np.ones(self.im_size)
else:
return self._var_map
def var_scalar(self):
if empty(self._var_scalar):
return self._default_var_scalar
else:
return self._var_scalar
def x2c(self):
if empty(self.corner_cutout):
return None
else:
return self.x2 + self.corner_section[1] - self.corner_cutout[1]
def calculate(self):
# these are the raw results from this subframe
self.size_section = self.image_section_proc.shape
if self.transposed:
self.size_section_tr = tuple(reversed(self.size_section))
else:
self.size_section_tr = self.size_section
if not empty(self.image): # part of the orignal image that is defined as the cutout (without processing!)
self.image_section_raw = self.image[self.corner_section[0]:self.size_section[0], self.corner_section[1]:self.size_section[1]]
self.radon_y1 = (2**(self.foldings-1)) * self.radon_max_idx[1] # size of each slice in y, times the slice number (index)
self.radon_y2 = (2**(self.foldings-1)) * (self.radon_max_idx[1]+1) # same thing, top index (should we include the last pixel?)
offset = (self.subframe.shape[2] - self.size_section_tr[1])//2 # this is added on either side if was_expanded
self.radon_x0 = self.radon_max_idx[2] - offset # position of x0 in the subframe (removing the expanded pixels)
self.radon_dx = self.radon_max_idx[0] - self.subframe.shape[0]//2 # the angle is in dim0 needs to offset for negative angles
# signal to noise from the subframe maximum
self.snr = self.subframe[self.radon_max_idx]
self.is_short = self.subframe.ndim>2 and self.subframe.shape[1]>1
# assume there is no transpose (then add it if needed)
self.y1 = self.radon_y1
self.y2 = self.radon_y2
self.dy = self.y2-self.y1
self.dx = self.radon_dx
if self.radon_dx!=0:
self.a = self.dy/self.dx
self.x0 = self.radon_x0 - self.y1/self.a
self.b = -self.a*self.x0
self.th = math.degrees(math.atan(self.a))
self.x1 = (self.y1-self.b)/self.a
self.x2 = (self.y2-self.b)/self.a
else:
self.a = float('NaN')
self.x0 = self.radon_x0
self.b = float('NaN')
self.th = 90
self.x1 = self.radon_x0
self.x2 = self.radon_x0
self.L = abs(self.radon_dy/math.sin(math.radians(self.th)))
# f = math.fabs(math.sin(math.radians(self.th)))
self.I = self.snr*math.sqrt(self.noise_var*2*math.sqrt(math.pi)*self.psf_sigma/(self.L))
self.snr_fwhm = self.I*0.81/math.sqrt(self.noise_var)
if self.transposed:
self.x1, self.y1 = self.y1, self.x1
self.x2, self.y2 = self.y2, self.x2
self.a = 1/self.a
self.b, self.x0 = self.x0, self.b
self.th = 90 - self.th
# self.x1 = round(self.x1)
# self.x2 = round(self.x2)
# self.y1 = round(self.y1)
# self.y2 = round(self.y2)
# self.x0 = round(self.x0)
############ calculate the errors on the Radon parameters #############
R = np.array(self.subframe[:,self.radon_max_idx[1],:]) # present this slice as 2D image
xmax = self.radon_max_idx[2]
ymax = self.radon_max_idx[0]
S = round(self.psf_sigma*self.num_psfs_peak_region) # size of area of a few PSF widths around the peak
x1 = xmax - S
if x1<0: x1 = 0
x2 = xmax + S
if x2>=R.shape[1]: x2 = R.shape[1]-1
y1 = ymax - S
if y1<0: y1 = 0
y2 = ymax + S
if y2>=R.shape[0]: y2 = R.shape[0]-1
x1 = int(x1)
x2 = int(x2)
y1 = int(y1)
y2 = int(y2)
C = np.array(R[y1:y2,x1:x2]) # make a copy of the array
xgrid,ygrid = np.meshgrid(range(C.shape[1]),range(C.shape[0]))
idx = np.unravel_index(np.nanargmax(C), C.shape)
mx = C[idx]
C[C<mx-self.num_snr_peak_region] = 0
xgrid = xgrid - idx[1]
ygrid = ygrid - idx[0]
self.radon_x_var = np.sum(C*xgrid**2)/np.sum(C)
self.radon_dx_var = np.sum(C*ygrid**2)/np.sum(C)
self.radon_xdx_cov = np.sum(C*xgrid*ygrid)/np.sum(C)
self.peak_region = C
self.makeCutout()
def mid_y_full(self): # this gives the middle y position from the full image
if empty(self.y1f) or empty(self.y2f):
return None
else:
return (self.y1f+self.y2f)/2
def mid_y(self):
if empty(self.y1) or empty(self.y2):
return None
else:
return (self.y1+self.y2)/2
def mid_x(self):
if empty(self.x1) or empty(self.x2):
return None
else:
return (self.x1+self.x2)/2
def y2c(self):
if empty(self.corner_cutout):
return None
else:
return self.y2 + self.corner_section[0] - self.corner_cutout[0]
def findSingle(self, M, transpose=False, threshold=None):
if empty(M):
return None
self.im_size = imsize(M)
if empty(threshold):
threshold = self.threshold
streak = None
self.num_frt_calls += 1
if self.use_short:
R_partial = FRT(
M, transpose=transpose, partial=True,
expand=False) # these are raw Radon partial transforms
# divide by the variance map, geometric factor, and PSF norm for each level
V = self.getRadonVariance(transpose)
G = [
self.getGeometricFactor(m)
for m in range(2,
len(R_partial) + 2)
] # m counts the number of foldings, partials start at 2
P = self.getNormFactorPSF()
R_partial = [
R_partial[i] / np.sqrt(V[i] * G[i] * P)
for i in range(len(R_partial))
]
R = R_partial[-1][:, 0, :] # get the final Radon image as 2D map
snrs_idx = [np.nanargmax(r)
for r in R_partial] # best index for each folding
# snrs_max = np.array([r[i] for i,r in zip(snrs_idx,R_partial)]) # best SNR for each folding
snrs_max = np.array([np.nanmax(r) for r in R_partial
]) # best SNR for each folding
best_idx = np.nanargmax(snrs_max) # which folding has the best SNR
best_snr = snrs_max[
best_idx] # what is the best SNR of all foldings
if best_snr >= threshold and 2**best_idx >= self.min_length:
peak_coord = np.unravel_index(
snrs_idx[best_idx], R_partial[best_idx].shape
) # the x,y,z of the peak in that subframe
streak = self.makeStreak(snr=best_snr,
transpose=transpose,
threshold=threshold,
peak=peak_coord,
foldings=best_idx + 2,
subframe=R_partial[best_idx],
section=M)
else:
R = FRT(M, transpose=transpose, partial=False, expand=True)
V = self.getRadonVariance(transpose)
foldings = len(
V) + 1 # the length tells you how many foldings we need
V = V[-1][:, 0, :] # get the last folding and flatten it to 2D
G = self.getGeometricFactor(foldings)
P = self.getNormFactorPSF
R = R / np.sqrt(V * G * P)
R_partial = R[:, np.
newaxis, :] # this is how it would look from a partial transpose output
idx = np.argmax(R)
best_snr = R[idx]
peak_coord = np.unravel_index(idx, R.shape)
peak_coord = (
peak_coord[0], 0, peak_coord[1]
) # added zero for y start position that is often non-zero in the partial transforms
streak = self.makeStreak(snr=best_snr,
transpose=transpose,
threshold=threshold,
peak=peak_coord,
foldings=foldings,
subframe=R_partial,
section=M)
self.best_SNR = max(
best_snr, self.best_SNR
) # this will always have the best S/N until "clear" is called
if not empty(streak):
self.streaks.append(streak)
if self.use_write_cutouts:
if empty(self.max_length_to_write
) or streak.L < self.max_length_to_write:
streak.write_to_disk()
# store the final FRT result (this is problematic once we start iterating over findSingle!)
if transpose:
self.radon_image_trans = R
else:
self.radon_image = R
if self.debug_bit > 1:
print(
"Running FRT %d times, trans= %d, thresh= %f, found streak: %d"
%
(self.num_frt_calls, transpose, threshold, not empty(streak)))
return streak
def FRT(M_in,
transpose=False,
expand=False,
padding=True,
partial=False,
output=None):
""" Fast Radon Transform (FRT) of the input matrix M_in (must be 2D numpy array)
Additional arguments:
-transpose (False): transpose M_in (replace x with y) to check all the other angles.
-expand (False): adds zero padding to the sides of the passive axis to allow for corner-crossing streaks
-padding (True): adds zero padding to the active axis to fill up powers of 2.
-partial (False): use this to save second output, a list of Radon partial images (useful for calculating variance at different length scales)
-output (None): give the a pointer to an array with the right size, for FRT to put the return value into it.
Note that if partial=True then output must be a list of arrays with the right dimensions.
"""
# print("running FRT with: transpose= "+str(transpose)+", expand= "+str(expand)+", padding= "+str(padding)+", partial= "+str(partial)+", finder= "+str(finder))
############### CHECK INPUTS AND DEFAULTS #################################
if empty(M_in):
return
if M_in.ndim > 2:
raise Exception("FRT cannot handle more dimensions than 2D")
############## PREPARE THE MATRIX #########################################
M = np.array(M_in) # keep a copy of M_in to give to finalizeFRT
np.nan_to_num(M, copy=False) # get rid of NaNs (replace with zeros)
if transpose:
M = M.T
if padding:
M = padMatrix(M)
if expand:
M = expandMatrix(M)
############## PREPARE THE MATRIX #########################################
Nfolds = getNumLogFoldings(M)
(Nrows, Ncols) = M.shape
M_out = []
if not empty(output): # will return the entire partial transform list
if partial:
for m in range(2, Nfolds + 1):
if output[m - 1].shape != getPartialDims(M, m):
raise RuntimeError("Wrong dimensions of output array["+str(m-1)+"]: "\
+str(output[m-1].shape)+", should be "+str(getPartialDims(M,m)))
M_out = output
else:
if output.shape[0] != 2 * M.shape[0] - 1:
raise RuntimeError("Y dimension of output ("+str(output.shape[0])+\
") is inconsistent with (padded and doubled) input ("+str(M.shape[0]*2-1)+")")
if output.shape[1] != M.shape[1]:
raise RuntimeError("X dimension of output ("+str(output.shape[1])+\
") is inconsistent with (expanded?) input ("+str(M.shape[1])+")")
dx = np.array([0], dtype='int64')
M = M[np.newaxis, :, :]
for m in range(1, Nfolds + 1): # loop over logarithmic steps
M_prev = M
dx_prev = dx
Nrows = M_prev.shape[1]
max_dx = 2**(m) - 1
dx = range(-max_dx, max_dx + 1)
if partial and not empty(output):
M = M_out[m -
1] # we already have memory allocated for this result
else:
M = np.zeros((len(dx), Nrows // 2, Ncols),
dtype=M.dtype) # make a new array each time
counter = 0
for i in range(Nrows //
2): # loop over pairs of rows (number of rows in new M)
for j in range(len(dx)): # loop over different shifts
# find the value and index of the previous shift
dx_in_prev = int(float(dx[j]) / 2)
j_in_prev = dx_in_prev + int(len(dx_prev) / 2)
# print "dx[%d]= %d | dx_prev[%d]= %d | dx_in_prev= %d" % (j, dx[j], j_in_prev, dx_prev[j_in_prev], dx_in_prev)
gap_x = dx[j] - dx_in_prev # additional shift needed
M1 = M_prev[j_in_prev, counter, :]
M2 = M_prev[j_in_prev, counter + 1, :]
M[j, i, :] = shift_add(M1, M2, -gap_x)
counter += 2
if partial and empty(
output
): # only append to the list if it hasn't been given from the start using "output"
M_out.append(M)
# end of loop on m
if not partial: # we don't care about partial transforms, we were not given an array to fill
# M_out = np.transpose(M, (0,2,1))[:,:,0] # lose the empty dimension
M_out = M[:, 0, :] # lose the empty dim
if not empty(output): # do us a favor and also copy it into the array
np.copyto(output, M_out)
# this can be made more efficient if we use the "output" array as
# target for assignment at the last iteration on m.
# this will save an allocation and a copy of the array.
# however, this is probably not very expensive and not worth
# the added complexity of the code
return M_out
def input(self,
image,
variance=None,
psf=None,
filename=None,
batch_num=None):
"""
Input an image and search for streaks in it.
Inputs: -images (expect numpy array, can be 3D)
-variance (can be scalar or map of noise variance)
-psf: point spread function of the image (scalar gaussian sigma or map)
-filename: for tracking the source of discovered streaks
-batch_num: if we are running many batches in this run
"""
if empty(image):
raise Exception("Cannot do streak finding without an image!")
self.clear()
if self.use_crop:
image = crop2size(image, self.crop_size)
self.im_size = imsize(image)
self.image = image
self.filename = filename
# input the variance, if given!
if not empty(variance):
if not scalar(variance) and self.use_crop:
self.input_var = crop2size(variance, self.crop_size)
else:
self.input_var = variance
# input the PSF if given!
if not empty(psf):
self.input_psf = psf
# housekeeping
self.filename = filename
self.batch_num = batch_num
corners = []
if self.use_sections:
sections = jigsaw(image,
self.size_sections,
output_corners=corners)
else:
sections = image[np.newaxis, ...]
for i in range(sections.shape[0]):
if not empty(corners):
self.current_section_corner = corners[i]
sec = sections[i, :, :]
# (m,v) = image_stats(sec)
sec = self.preprocess(sec)
self.scanThresholds(sec)
if self.use_show:
plt.clf()
h = plt.imshow(self.image)
h.set_clim(0, 5 * np.sqrt(self.var_scalar))
[streak.plotLines(im_type='full') for streak in self.streaks]
plt.title("full frame image")
plt.xlabel(self.filename)
f = plt.gcf()
f.canvas.draw()
f.canvas.flush_events()
def frt(M_in,
transpose=False,
expand=False,
padding=True,
partial=False,
finder=None,
output=None):
""" Fast Radon Transform (FRT) of the input matrix M_in (must be 2D numpy array)
Additional arguments:
-transpose (False): transpose M_in (replace x with y) to check all the other angles.
-expand (False): adds zero padding to the sides of the passive axis to allow for corner-crossing streaks
-padding (True): adds zero padding to the active axis to fill up powers of 2.
-partial (False): use this to save second output, a list of Radon partial images (useful for calculating variance at different length scales)
-finder (None): give a "finder" object that is used to scan for streaks. Must have a "Scan" method.
-output (None): give the right size array for FRT to put the return value into it.
"""
# print "running FRT with: transpose= "+str(transpose)+", expand= "+str(expand)+", padding= "+str(padding)+", partial= "+str(partial)+", finder= "+str(finder)
############### CHECK INPUTS AND DEFAULTS #################################
if empty(M_in):
return
if M_in.ndim > 2:
raise Exception("FRT cannot handle more dimensions than 2D")
if not empty(finder):
if not isinstance(finder, pyradon.finder.Finder):
raise Exception(
"must input a pyradon.finder.Finder object as finder...")
scan_method = getattr(finder, 'scan', None)
if not scan_method or not callable(scan_method):
raise Exception("finder given to FRT doesn't have a scan method")
finder.last_streak = []
############## PREPARE THE MATRIX #########################################
M = np.array(M_in) # keep a copy of M_in to give to finalizeFRT
if transpose:
M = M.T
if not empty(finder):
finder._im_size_tr = M.shape
finder.im_size = M_in.shape
if padding:
M = padMatrix(M)
if expand:
M = expandMatrix(M)
M_partial = []
(Nrows, Ncols) = M.shape
dx = np.array([0])
M = M[np.newaxis, :, :]
for m in range(1,
int(math.log(Nrows, 2)) + 1): # loop over logarithmic steps
M_prev = M
dx_prev = dx
Nrows = M.shape[1] # number of rows in M_prev!
max_dx = 2**(m) - 1
dx = range(-max_dx, max_dx + 1)
M = np.zeros((len(dx), Nrows / 2, Ncols), dtype=M.dtype)
counter = 0
for i in range(
Nrows /
2): # loop over pairs of rows (number of rows in new M)
for j in range(len(dx)): # loop over different shifts
# find the value and index of the previous shift
dx_in_prev = int(float(dx[j]) / 2)
j_in_prev = dx_in_prev + int(len(dx_prev) / 2)
# print "dx[%d]= %d | dx_prev[%d]= %d | dx_in_prev= %d" % (j, dx[j], j_in_prev, dx_prev[j_in_prev], dx_in_prev)
gap_x = dx[j] - dx_in_prev # additional shift needed
M1 = M_prev[j_in_prev, counter, :]
M2 = M_prev[j_in_prev, counter + 1, :]
M[j, i, :] = shift_add(M1, M2, -gap_x)
counter += 2
if finder:
finder.scan(M, transpose)
if partial:
M_partial.append(M)
# end of loop on m
M_out = np.transpose(M, (0, 2, 1))[:, :, 0] # lose the empty dimension
if not empty(finder):
finder.finalizeFRT(M_in, transpose, M_out)
if partial:
if not empty(output):
np.copyto(output, M_partial)
return M_partial
else:
if not empty(output):
np.copyto(output, M_out)
return M_out
