def median_bins_fits(images, B):
mn, sd = running_stats(images)
dim = mn.shape
left_bin = np.zeros(dim)
bins = np.zeros((dim[0], dim[1], B))
bin_width = 2 * sd / B
minval = mn - sd
maxval = mn + sd
for image in images:
HDUlist = fits.open(image)
dat = HDUlist[0].data
for i in range(dim[0]):
for j in range(dim[1]):
value = dat[i, j]
if value < minval[i, j]:
left_bin[i, j] += 1
elif value >= minval[i, j] and value < maxval[i, j]:
bin = int((value - (minval[i, j])) / bin_width[i, j])
bins[i, j, bin] += 1
return (mn, sd, left_bin, bins)
def median_bins_fits(filenames, B):
# Calculate the mean and standard dev
mean, std = running_stats(filenames)
dim = mean.shape # Dimension of the FITS file arrays
# Initialise bins
left_bin = np.zeros(dim)
bins = np.zeros((dim[0], dim[1], B))
bin_width = 2 * std / B
# Loop over all FITS files
for filename in filenames:
hdulist = fits.open(filename)
data = hdulist[0].data
# Loop over every point in the 2D array
for i in range(dim[0]):
for j in range(dim[1]):
value = data[i, j]
mean_ = mean[i, j]
std_ = std[i, j]
if value < mean_ - std_:
left_bin[i, j] += 1
elif value >= mean_ - std_ and value < mean_ + std_:
bin = int((value - (mean_ - std_)) / bin_width[i, j])
bins[i, j, bin] += 1
return mean, std, left_bin, bins
def median_bins_fits(fileNameArray, B):
if len(fileNameArray) == 0:
return 0, 0, 0, 0
firstFileName = fits.open(fileNameArray[0])
firstFileData = firstFileName[0].data
numberOfRows = len(firstFileData)
numberOfColumns = len(firstFileData[0])
mu, sigma = running_stats(fileNameArray)
minval = mu - sigma
maxval = mu + sigma
width = 2 * sigma / B
numberOfSmallerValues = np.zeros((numberOfRows, numberOfColumns))
binCountArray = np.zeros((numberOfRows, numberOfColumns, B))
for fileName in fileNameArray:
hdulist = fits.open(fileName)
data = hdulist[0].data
for index, value in np.ndenumerate(data):
if value < minval[index]:
numberOfSmallerValues[index] += 1
elif value >= maxval[index]:
continue
else:
binIndex = int((value - minval[index]) // width[index])
binCountArray[index][binIndex] += 1
return mu, sigma, numberOfSmallerValues, binCountArray
def median_bins_fits(images, bins):
mean, stdv = running_stats(images)
a = get_data(images)
minval = mean - stdv
maxval = mean + stdv
width = 2 * stdv / bins
left = a[0:len(a)] < minval
left_bin = np.sum(left, axis=(0))
in_bins = []
x = minval
i = 0
while i < bins:
passend = (a[0:len(a)] >= x) & (a[0:len(a)] < x + width)
s = np.sum(passend, axis=(0))
if in_bins == []:
in_bins = s
else:
in_bins = np.dstack((in_bins, s))
x += width
i += 1
return (mean, stdv, left_bin, np.array(in_bins))
def median_bins_fits(filenames, nbins):
#main variables (returns)
mean, stdev = running_stats(filenames) #each has 200x200 dimensions
n_smaller = np.zeros(mean.shape)
bin_counts = np.zeros((mean.shape[0], mean.shape[1], nbins))
minval = mean - stdev #matrix operation
width = 2 * stdev / nbins
# process every image one at a time
for fitsfile in filenames:
hdulist = fits.open(fitsfile)
image = hdulist[0].data
# for every pixel in the image
for x in range(0, mean.shape[0]):
for y in range(0, mean.shape[1]):
binmax = minval[x, y]
value = image[x, y]
if value < minval[x, y]: #skip this value
n_smaller[x, y] += 1
else: # fit into a bin
for index in range(nbins):
binmax += width[x, y]
if value < binmax: #values > maxval -> ignored
bin_counts[x, y, index] += 1
break
return (mean, stdev, n_smaller, bin_counts)
def median_bins_fits(paths, B):
mean, std = running_stats(paths)
minval = mean - std
bin_width = 2 * std / B
bins = np.zeros((mean.shape[0], mean.shape[1], B))
count = np.zeros(mean.shape)
for path in paths:
data = fits.open(path)[0].data
for i in range(data.shape[0]):
for j in range(data.shape[1]):
if data[i, j] < minval[i, j]:
count[i, j] += 1
elif data[i, j] < mean[i, j] + std[i, j]:
bins[i, j,
int((data[i, j] - (mean[i, j] - std[i, j])) /
bin_width[i, j])] += 1
return mean, std, count, bins
def median_bins_fits (fitslst, B) :
means, stds = running_stats (fitslst)
minvals = means - stds
bwidths = 2*stds/B
bins = np.zeros (shape = (means.shape[0], means.shape[1], B))
ignores = np.zeros (shape = means.shape)
for file in fitslst :
hdulist = fits.open (file)
data = hdulist[0].data
belmin = data < minvals
ignores = ignores + belmin.astype(int)
zidx = np.floor((data - minvals)/bwidths).astype (int)
for i in range (0, B) :
bins[...,i] += (zidx == i).astype(int)
return (means, stds, ignores, bins)
def median_bins_fits(files, bin_number):
mean, std = running_stats(files)
x, y = mean.shape
min_val, max_val = (mean - std), (mean + std)
bin_width = 2 * std / bin_number
bin_counts = np.zeros([x, y, bin_number + 1])
bin_ignores = np.zeros([x, y])
for file in files:
data = fits.open(file)[0].data
bin_ignores += 1 * (data < min_val)
for i in range(1, bin_number + 1):
bin_counts[:, :, i] += 1 * ((data < (min_val + i * bin_width)) &
(data > min_val + (i - 1) * bin_width))
return mean, std, bin_ignores, bin_counts[:, :, 1:]
def median_bins_fits(files, B):
mean, std = running_stats(files)
dim = mean.shape
left_bin = np.zeros(dim)
bins = np.zeros((dim[0], dim[1], B))
bin_width = 2 * std / B
for _file in files:
hdulist = fits.open(_file)
data = hdulist[0].data
for i in range(dim[0]):
for j in range(dim[1]):
value = data[i, j]
_mean = mean[i, j]
_std = std[i, j]
if value < _mean - _std:
left_bin[i, j] += 1
elif value < _mean + _std:
_bin = int((value - (_mean - _std)) / bin_width[i, j])
bins[i, j, _bin] += 1
return mean, std, left_bin, bins
def median_bins_fits(fits_files, B):
num_files = len(fits_files)
# Load FITS data into 3-d numpy array
data = np.zeros((NX, NY, num_files))
for i in range(0, num_files):
hdulist = fits.open(fits_files[i])
data[:, :, i] = hdulist[0].data
# Initialize bin counts, one for each bin for each pixel
bin_counts = np.zeros((NX, NY, B))
# Calculate per-pixel mean, std deviations (NX x NY arrays)
(means, stds) = running_stats(fits_files)
# Calculate per-pixel bin boundaries/widths
minvals = means - stds
bin_widths = 2 * stds / B
# Per-pixel number of values in left (ignore) bin, this is the number of data points
# with values less than the mean - std dev for each pixel.
left_bin_counts = sum(
[data[:, :, i] < minvals for i in range(0, num_files)])
left_bin_counts = np.array(left_bin_counts, float)
# Calculate bin counts
for i in range(0, B):
bin_mins = minvals + i * bin_widths
bin_maxs = minvals + (i + 1) * bin_widths
for j in range(0, num_files):
in_this_bin = np.logical_and(data[:, :, j] >= bin_mins,
data[:, :, j] < bin_maxs)
bin_counts[:, :, i] += in_this_bin
# return results
return means, stds, left_bin_counts, bin_counts
def median_bins_fits(file_list, N):
mean, std = running_stats(file_list)
dim = mean.shape
ignore_bin = np.zeros(dim)
bin_array = np.zeros((dim[0], dim[1], N))
#print(bin_array)
width = 2*std/N
#print(dim)
#print(dim[0], dim[1], N)
for item in file_list:
hdulist = fits.open(item)
data= hdulist[0].data
for i in range(dim[0]):
for j in range(dim[1]):
value = data[i,j]
mu = mean[i,j]
sigma = std[i,j]
if value<(mu-sigma):
ignore_bin[i,j] +=1
elif value>=(mu-sigma) and value<(mu+sigma):
bin_number = int((value - (mu-sigma))/width[i,j])
bin_array[i, j, bin_number]+=1
return mean, std, ignore_bin, bin_array
def median_bins_fits(filenames, B):
mean, std = running_stats(filenames)
dim = mean.shape
left_bin = np.zeros(dim)
bins = np.zeros((dim[0], dim[1], B))
bin_width = 2 * std / B
for filename in filenames:
hdulist = fits.open(filename)
data = hdulist[0].data
for i in range(dim[0]):
for j in range(dim[1]):
value = data[i, j]
mean_ = mean[i, j]
std_ = std[i, j]
if value < mean_ - std_:
left_bin[i, j] += 1
elif value >= mean_ - std_ and value < mean_ + std_:
bin = int((value - (mean_ - std_)) / bin_width[i, j])
bins[i, j, bin] += 1
return mean, std, left_bin, bins
def median_bins_fits(list_of_files, B):
##@@@@@@@@ NB I've blanked out the actual bins (costly)
# Get & numericise the data
new_list_files = []
for file in list_of_files:
hdulist = fits.open(file)
data = hdulist[0].data
new_list_files.append(data)
# Definitions for iterating over pixels
n_rows = new_list_files[0].shape[0]
n_cols = new_list_files[0].shape[1]
# Vectorized mean, stdev and bounds
mean_ = running_stats(list_of_files)[0] #np.mean(values)
stdev = running_stats(list_of_files)[1] #np.std(values)
l_bound = mean_ - stdev
u_bound = mean_ + stdev
# Vectorized "too-small" count
## fail (returns 1, shd be 0): too_small = np.where(values < l_bound)
too_small = sum(new_list_files < l_bound)
# Pixel loop to calculate bin counts for each pixel
width = stdev * 2 / B # this is still vectorized
#@@@@@@@@@@@ bins_array = np.zeros((n_rows,n_cols,B), dtype=object)
count_array = np.zeros((n_rows, n_cols, B), dtype=object)
# Start pixel loop
for row in range(n_rows): # 0,1,2, ..., n_rows-1
for col in range(n_cols):
counts_list = [] # the required counts (1-D list)
temp_file_list = new_list_files # VECTOR to preserve original (costly)
#@@@@@@@@@ bins=[] # the actual elements go here
for i in range(1, B + 1):
inner_list = [
] # the actual bin, contains elements (update array with this)
local_count = 0 # bin size
for image in temp_file_list:
w = width[row, col]
lb = l_bound[row, col] + (i - 1) * w
ub = lb + w
val = image[row, col]
if lb <= val < ub: # if that pixel is in interval...
local_count += 1 # increase bin size
count_array[row, col, i - 1] += 1
#@@@@@@@@@@ inner_list.append(val) # add el to bin
# temp_values.remove(val) this causes skipping! use copy.copy...
counts_list.append(
local_count) # this is know the answer but a list
#@@@@@@@@@@@@@@@ bins.append(inner_list)
# Exit bin-size loop
# Update original pixel map
#@@@@@@@@@@@@@@@bins_array[row,col,i-1] = bins @@@@@@@@@@@
#bins_array[row,col,i-1] = np.asarray(inner_list) #exceeds CPU limit ##recent
#counts_list = np.asarray(counts_list)
#############count_array[row,col,i-1] = counts_list[i-1]
#count_array[row,col,i-1] = np.asarray(counts_list) #exceeds CPU limit
# End pixel loop
# Convert both to array - redundant following code edits above
#count_array = np.asarray(count_array)
#bins = np.asarray(bins)
# Outputs
return mean_, stdev, too_small, count_array