def classify(melanoma, ground, feature, classifier, block=True):
seg = []
tim = []
dim = []
for (melanoma_item, ground_item) in zip(melanoma, ground):
print('Segmentating...')
print('\t' + melanoma_item)
img = Image(cfg.melanoma_path + melanoma_item,
cfg.ground_path + ground_item, cfg.block)
size = img.get_shape()
portion = img.get_portion()
dim.append(size)
img_seg = np.zeros((size[0], size[1]))
row = [portion, size[0] - portion]
col = [portion, size[1] - portion]
if block:
st = time.time()
seg.append(per_block(img, img_seg, row, col, feature, classifier))
tim.append(time.time() - st)
else:
st = time.time()
seg.append(per_pixel(img, img_seg, row, col, feature, classifier))
tim.append(time.time() - st)
return seg, tim, dim
def flat(image_id):
filename, cam = get_cam_unzipped_filename(image_id)
flatfile = os.path.join(FLATPATH, flat_d[cam])
image = Image(filename)
name, ext = os.path.splitext(filename)
out = name + ".flat" + ext
image.divide(flatfile, out)
sql = "REPLACE INTO flat VALUES (%d, '%s', '%s')" % \
(image_id, flatfile, out)
run_sql(sql)
def __init__(self,
image: (Image, str, Path),
simulation: Simulation,
max_dist: float = 0.5,
bin_size: float = 1):
"""
Constructor
:param image:
An Image object, or a path to a FITS image
:param simulation:
The parsed content of an Astromatic stuff simulation
:param max_dist:
Two objects are considered a cross-match if they are closer than this number of pixels
:param bin_size:
Bin size (in magnitude units) used for the histogram of stars and galaxies
"""
if isinstance(image, str) or isinstance(image, Path):
self.__image = Image(image)
else:
self.__image = image
self.__max_dist = max_dist
self.stars = self.__image.get_contained_sources(
simulation.stars.ra,
simulation.stars.dec,
mag=simulation.stars.mag,
flux=simulation.stars.flux)
self.galaxies = self.__image.get_contained_sources(
simulation.galaxies.ra,
simulation.galaxies.dec,
mag=simulation.galaxies.mag,
flux=simulation.galaxies.flux)
self.__all_mag = np.append(self.stars.mag, self.galaxies.mag)
all_x = np.append(self.stars.x, self.galaxies.x)
all_y = np.append(self.stars.y, self.galaxies.y)
self.__kdtree = KDTree(np.column_stack([all_x, all_y]))
all_mags = np.append(self.stars.mag, self.galaxies.mag)
self.__min_mag = np.floor(np.min(all_mags))
self.__max_mag = np.ceil(np.max(all_mags))
self.__bin_size = bin_size
self.__edges = np.arange(self.__min_mag - bin_size / 2.,
self.__max_mag + bin_size / 2., bin_size)
self.stars_bins, _ = np.histogram(self.stars.mag, bins=self.__edges)
self.galaxies_bins, _ = np.histogram(self.galaxies.mag,
bins=self.__edges)
def coadded_frame_cross(coadded_catalog, sim12_r_simulation, datafiles, tolerances):
image = Image(
datafiles / 'sim12' / 'img' / 'sim12_r.fits.gz',
weight_image=datafiles / 'sim12' / 'img' / 'sim12_r.weight.fits.gz'
)
cross = CrossMatching(image, sim12_r_simulation, max_dist=tolerances['distance'])
return cross(coadded_catalog['pixel_centroid_x'], coadded_catalog['pixel_centroid_y'])
def second_method(self, melanoma, ground):
"""
Implements the second method of feature extraction. In this method I consider a better params to create the
gabor kernels.
Parameters
----------
melanoma: list
A list of strings, which contents is the path of melanoma images
ground: list
A list of strings, which contents is a the path of ground images
Returns
-------
A data set X and y, which contents data about the images
"""
X = np.zeros((cfg.nImage * cfg.nSample, cfg.nCells - 1))
y = np.zeros((cfg.nImage * cfg.nSample,))
self.kernels = self.second_gabor_bank(fmax=cfg.fmax, ns=cfg.ns, nd=cfg.nd, v=cfg.v, b=cfg.b)
self.values = second_values
for (melanoma_item, ground_item, image_index) in zip(melanoma, ground, range(cfg.nImage)):
img = Image(cfg.melanoma_path + melanoma_item, cfg.ground_path + ground_item, cfg.block)
for sample_index in range(cfg.nSample):
index = image_index * cfg.nSample + sample_index
X[index, :], y[index] = second_features(img, self.kernels)
return X, y.astype(int)
def task(filename, taskParams):
'''
@author
@param task_params -
@return
'''
with Image(filename) as img:
# Create the region object
regionType, regionValue = taskParams["region"]["type"], taskParams[
"region"]["value"]
region = Region(regionType, regionValue)
rowStart = taskParams["region"]["value"]["y1"]
rpb = taskParams["rowsperbin"]
# Calculates projection for group of rows
def calcGroupProj(data, start):
end = start + rpb
return np.mean(data[start:end, :])
# High level function to calculate all projections
def calcRegionProj(data):
projs = []
for i in range(0, data.shape[0], rpb):
value = calcGroupProj(data, i)
projs.append([float(value), rowStart + i, rowStart + i + rpb])
return projs
projs = region.execute(img, calcRegionProj)
#projs[-1][2] = taskParams["region"]["value"]["y2"]
return {"projs": projs}, None
def first_method(self, melanoma, ground):
"""
Implements a first method of feature extraction.
This method uses some techinques with random parameteres.
With this method I am going to get some aproximation about how good this working is.
Parameters
----------
melanoma: list
A list of strings, which contents is the path of melanoma images
ground: list
A list of strings, which contents is a the path of ground images
Returns
-------
A data set X and y, which contents data about the images
"""
n = len(melanoma)
X = np.zeros((n * cfg.nSample, cfg.nCells - 1))
y = np.zeros((n * cfg.nSample,))
self.kernels = self.first_gabor_bank(cfg.gabor_params)
self.values = first_values
for (melanoma_item, ground_item, image_index) in zip(melanoma, ground, range(cfg.nImage)):
img = Image(cfg.melanoma_path + melanoma_item, cfg.ground_path + ground_item, cfg.block)
for sample_index in range(cfg.nSample):
index = image_index * cfg.nSample + sample_index
X[index, :], y[index] = first_features(img, self.kernels)
return X, y.astype(int)
def g_cross(modelfitting_catalog, sim12_g_simulation, datafiles, tolerances):
image = Image(datafiles / 'sim12' / 'img' / 'sim12.fits.gz',
weight_image=datafiles / 'sim12' / 'img' /
'sim12.weight.fits.gz')
cross = CrossMatching(image,
sim12_g_simulation,
max_dist=tolerances['distance'])
return cross(modelfitting_catalog['pixel_centroid_x'],
modelfitting_catalog['pixel_centroid_y'])
def multi_compressed_cross(multi_compressed_catalog, sim12_r_simulation,
datafiles, tolerances):
image = Image(datafiles / 'sim12' / 'img' / 'sim12_r.compressed.fits',
weight_image=datafiles / 'sim12' / 'img' /
'sim12_r.weight.compressed.fits')
cross = CrossMatching(image,
sim12_r_simulation,
max_dist=tolerances['distance'])
return cross(multi_compressed_catalog['pixel_centroid_x'],
multi_compressed_catalog['pixel_centroid_y'])
def task(filename, taskParams):
'''
@author Joe Pagliuco
@modified Wei Ren
@param taskParams
@return data for general histogram function
'''
with Image(filename) as img:
# Create the region object
regionType, regionValue = taskParams["region"]["type"], taskParams[
"region"]["value"]
region = Region(regionType, regionValue)
regionMin = float(region.execute(img, np.min))
regionMax = float(region.execute(img, np.max))
numBins, minValue, maxValue = taskParams["bins"], taskParams[
"min"], taskParams["max"]
_rangeMin = regionMin if minValue == "auto" else float(minValue)
_rangeMax = regionMax if maxValue == "auto" else float(maxValue)
if (_rangeMin > _rangeMax):
_rangeMin, _rangeMax = _rangeMax, _rangeMin
# Largest floating point number
maxFloat = np.finfo(float).max
def histogram(a):
# Good estimator for number of bins
_numBins = numBins if numBins != "auto" else "doane"
return np.histogram(a, bins=_numBins, range=(_rangeMin, _rangeMax))
def underflow(a):
underflowBin = np.where(a.flatten() < _rangeMin)[0]
return underflowBin.size
def overflow(a):
overflowBin = np.where(a.flatten() > _rangeMax)[0]
return overflowBin.size
# Gives us the histogram for the values in the range
h = region.execute(img, histogram)
u = region.execute(img, underflow)
o = region.execute(img, overflow)
fireflyHist = NumpyToFireflyHist(h)
binWidth = float(h[1][1] - h[1][0])
underflow = [[float(u), _rangeMin - binWidth, _rangeMin]]
overflow = [[float(o), _rangeMax, _rangeMax + binWidth]]
return {
"main": fireflyHist,
"underflow": underflow,
"overflow": overflow
}, None
def build_array(self, data, ground, data_path, ground_path):
X = np.zeros((cfg.nSample * cfg.nImage, cfg.advanced_n_cells))
y = np.zeros((cfg.nSample * cfg.nImage,))
for (data_name, ground_name, image_index) in zip(data, ground, range(cfg.nImage)):
img = Image(data_path + data_name, ground_path + ground_name, cfg.blockDim)
self.set_theta(ground_path + ground_name, True)
self.set_kernel()
for sample_index in range(cfg.nSample):
index = image_index * cfg.nSample + sample_index
feats = self.features(img)
X[index, 0:15] = feats[0]
X[index, 15] = feats[1]
y[index] = feats[2]
return X, y.astype(int)
def image_sort_example():
"""
Sort a bunch of images based on different keys
"""
files = darkfiles("sbc")
images = valid_images(files)
image = Image(files[0])
print image.MIN
print image.BSCALE
sortby(images, "MAX")
print[image.MAX for image in images]
sortby(images, "MEAN")
print[image.MEAN for image in images]
def task(filename, taskParams):
'''
@author
@param task_params -
@return
'''
data = ["Amplifier, Data, Pre, Post, Over"]
with Image(filename) as img:
for r in taskParams["regions"]:
entry = r["name"] + ","
entry += str(_calcNoise(img, r["data"])) + ","
entry += str(_calcNoise(img, r["pre"])) + ","
entry += str(_calcNoise(img, r["post"])) + ","
entry += str(_calcNoise(img, r["over"]))
data.append(entry)
return {"data": data}, None
def task(filename, taskParams):
''' Return a hot pixel in the defined region.
@author
@param task_params -
@return
'''
threshold = taskParams["threshold"]
regionParam = taskParams["region"]
region = Region(regionParam["type"], regionParam["value"])
with Image(filename) as img:
maxOfRegion = region.execute(img, np.max)
threshold = maxOfRegion if threshold == "max" else float(threshold)
# For a rectangular region
def findHotPixelsRect(data):
hotPixels = []
max = 500
x, y = regionParam["value"]["x1"], regionParam["value"]["y1"]
for index, value in np.ndenumerate(data):
# If it breaks the threshold, add it to the list, but offset the location
# to match the location
if value >= threshold:
hotPixels.append({"x": index[1] + x, "y": index[0] + y})
# Max number of hot pixels
if len(hotPixels) > max - 1:
break
return {"hotPixels": hotPixels}, None
# For a circular region
def findHotPixelsCirc(data):
pass
return region.execute(img, findHotPixelsRect, findHotPixelsCirc)
return {"error:", "Error reading image file"}
class CrossMatching(object):
"""
This class helps cross-matching objects from the simulation and from a catalog
using an image to retrieve non-matched objects (objects from the simulation that are
within the image but did not match any from the catalog)
"""
class CrossMatchResult(object):
"""
Holds the result from a catalog/image/simulation cross match
"""
def __init__(self, stars_found: np.recarray, stars_missed: np.recarray,
stars_recall: [float], stars_catalog: [int],
galaxies_found: np.recarray, galaxies_missed: np.recarray,
galaxies_recall: [float], galaxies_catalog: [int],
misids: [float]):
"""
Constructor
:param stars_found:
Information about the true matched stars
:param stars_missed:
Information about the true non-matched stars
:param stars_recall:
Histogram of star recall (by magnitude)
:param stars_catalog:
Catalog indexes that correspond to true stars
:param galaxies_found:
Information about the true matched galaxies
:param galaxies_missed:
Information about the true non-matched galaxies
:param galaxies_recall:
Histogram of galaxy recall (by magnitude)
:param galaxies_catalog:
Catalog indexes that correspond to true galaxies
:param misids:
Histogram of mis-identifications (by magnitude)
"""
self.stars_found = stars_found
self.stars_not_found = stars_missed
self.stars_recall = stars_recall
self.stars_catalog = stars_catalog
self.galaxies_found = galaxies_found
self.galaxies_not_found = galaxies_missed
self.galaxies_recall = galaxies_recall
self.galaxies_catalog = galaxies_catalog
self.misids = misids
@property
def all_catalog(self):
"""
:return: All indexes from the catalog that have been matches, regardless of the type.
:note: Use stars_catalog or galaxies_catalog if you want to check for attributes specific to one or the
other (i.e. there is no bulge for stars)
"""
return np.append(self.stars_catalog, self.galaxies_catalog)
@property
def all_magnitudes(self):
"""
:return: All magnitudes, from matched stars and galaxies
"""
return np.append(self.stars_found.mag, self.galaxies_found.mag)
@property
def all_fluxes(self):
"""
:return: All fluxes, from matched stars and galaxies
"""
return np.append(self.stars_found.flux, self.galaxies_found.flux)
def __init__(self,
image: (Image, str, Path),
simulation: Simulation,
max_dist: float = 0.5,
bin_size: float = 1):
"""
Constructor
:param image:
An Image object, or a path to a FITS image
:param simulation:
The parsed content of an Astromatic stuff simulation
:param max_dist:
Two objects are considered a cross-match if they are closer than this number of pixels
:param bin_size:
Bin size (in magnitude units) used for the histogram of stars and galaxies
"""
if isinstance(image, str) or isinstance(image, Path):
self.__image = Image(image)
else:
self.__image = image
self.__max_dist = max_dist
self.stars = self.__image.get_contained_sources(
simulation.stars.ra,
simulation.stars.dec,
mag=simulation.stars.mag,
flux=simulation.stars.flux)
self.galaxies = self.__image.get_contained_sources(
simulation.galaxies.ra,
simulation.galaxies.dec,
mag=simulation.galaxies.mag,
flux=simulation.galaxies.flux)
self.__all_mag = np.append(self.stars.mag, self.galaxies.mag)
all_x = np.append(self.stars.x, self.galaxies.x)
all_y = np.append(self.stars.y, self.galaxies.y)
self.__kdtree = KDTree(np.column_stack([all_x, all_y]))
all_mags = np.append(self.stars.mag, self.galaxies.mag)
self.__min_mag = np.floor(np.min(all_mags))
self.__max_mag = np.ceil(np.max(all_mags))
self.__bin_size = bin_size
self.__edges = np.arange(self.__min_mag - bin_size / 2.,
self.__max_mag + bin_size / 2., bin_size)
self.stars_bins, _ = np.histogram(self.stars.mag, bins=self.__edges)
self.galaxies_bins, _ = np.histogram(self.galaxies.mag,
bins=self.__edges)
@property
def bin_centers(self):
"""
:return: The center of the bins used for the magnitude histograms
"""
return self.__bin_size / 2. * (self.__edges[1:] + self.__edges[:-1])
def __call__(self, x: np.array, y: np.array, mag: np.array = None):
"""
Perform a cross-matching between a catalog and the simulation part contained within the image
:param x:
X coordinate, in pixels, of the detected object
:param y:
Y coordinate, in pixels, of the detected object
:param mag:
Optional, measured magnitude. This will be used to bin mis-identified sources, as they do not
have a "true" magnitude to do the binning.
:return:
A CrossMatchResult object containing the matches and mismatches
"""
nstars = len(self.stars)
ngalaxies = len(self.galaxies)
d, i = self.__kdtree.query(np.column_stack([x, y]))
dist_filter = (d <= self.__max_dist)
hits = np.recarray((len(x), ),
dtype=[('source', int), ('catalog', int),
('distance', float)])
hits['source'] = i
hits['catalog'] = np.arange(len(x))
hits['distance'] = d
hits = hits[dist_filter]
real_found = np.unique(hits['source'])
real_catalog = []
for r in real_found:
real_catalog.append(
np.sort(hits[hits['source'] == r],
order='distance')['catalog'][0])
real_catalog = np.asarray(real_catalog)
# Found contains now the index of the "real" stars and galaxies with at least one match
# If the index is < len(self.__stars), it is a star
stars_found = real_found[real_found < nstars]
stars_catalog = real_catalog[real_found < nstars]
stars_not_found = np.setdiff1d(np.arange(nstars), stars_found)
stars_hist, _ = np.histogram(self.stars.mag[stars_found],
bins=self.__edges)
stars_recall = np.divide(stars_hist,
self.stars_bins,
out=np.zeros(stars_hist.shape),
where=self.stars_bins != 0)
# If the index is > len(self.__stars, it is a galaxy)
galaxies_found = real_found[real_found >= nstars] - nstars
galaxies_catalog = real_catalog[real_found >= nstars]
galaxies_not_found = np.setdiff1d(np.arange(ngalaxies), galaxies_found)
galaxies_hist, _ = np.histogram(self.galaxies.mag[galaxies_found],
bins=self.__edges)
galaxies_recall = np.divide(galaxies_hist,
self.galaxies_bins,
out=np.zeros(galaxies_hist.shape),
where=self.galaxies_bins != 0)
# Detections that are too far from any "real" source
# We show them binned by measured, and by nearest
if mag is not None:
bad_filter = (d >= self.__max_dist)
bad_mag = mag[bad_filter]
bad_hist, _ = np.histogram(bad_mag, bins=self.__edges)
sum_hist = (galaxies_hist + stars_hist + bad_hist)
misids = np.divide(bad_hist,
sum_hist,
out=np.zeros(bad_hist.shape),
where=sum_hist != 0)
else:
misids = None
return CrossMatching.CrossMatchResult(
self.stars[stars_found], self.stars[stars_not_found], stars_recall,
stars_catalog, self.galaxies[galaxies_found],
self.galaxies[galaxies_not_found], galaxies_recall,
galaxies_catalog, misids)
