def array_value_range(a):
if len(a) == 0:
return (None, None)
from numpy import minimum, maximum
r = (minimum.reduce(a), maximum.reduce(a))
return r
def _array_value_range(a):
if len(a) == 0:
return (None, None)
from numpy import minimum, maximum
r = (minimum.reduce(a), maximum.reduce(a))
return r
def py_vq2(obs, code_book, check_finite=True):
"""2nd Python version of vq algorithm.
The algorithm simply computes the euclidian distance between each
observation and every frame in the code_book/
Parameters
----------
obs : ndarray
Expect a rank 2 array. Each row is one observation.
code_book : ndarray
Code book to use. Same format than obs. Should have same number of
features (eg columns) than obs.
check_finite : bool, optional
Whether to check that the input matrices contain only finite numbers.
Disabling may give a performance gain, but may result in problems
(crashes, non-termination) if the inputs do contain infinities or NaNs.
Default: True
Returns
-------
code : ndarray
code[i] gives the label of the ith obversation, that its code is
code_book[code[i]].
mind_dist : ndarray
min_dist[i] gives the distance between the ith observation and its
corresponding code.
Notes
-----
This could be faster when number of codebooks is small, but it
becomes a real memory hog when codebook is large. It requires
N by M by O storage where N=number of obs, M = number of
features, and O = number of codes.
"""
obs = _asarray_validated(obs, check_finite=check_finite)
code_book = _asarray_validated(code_book, check_finite=check_finite)
d = shape(obs)[1]
# code books and observations should have same number of features
if not d == code_book.shape[1]:
raise ValueError(
"""
code book(%d) and obs(%d) should have the same
number of features (eg columns)"""
% (code_book.shape[1], d)
)
diff = obs[newaxis, :, :] - code_book[:, newaxis, :]
dist = sqrt(np.sum(diff * diff, -1))
code = argmin(dist, 0)
min_dist = minimum.reduce(dist, 0)
# The next line I think is equivalent and should be faster than the one
# above, but in practice didn't seem to make much difference:
# min_dist = choose(code,dist)
return code, min_dist
def py_vq2(obs, code_book, check_finite=True):
"""2nd Python version of vq algorithm.
The algorithm simply computes the euclidian distance between each
observation and every frame in the code_book/
Parameters
----------
obs : ndarray
Expect a rank 2 array. Each row is one observation.
code_book : ndarray
Code book to use. Same format than obs. Should have same number of
features (eg columns) than obs.
check_finite : bool, optional
Whether to check that the input matrices contain only finite numbers.
Disabling may give a performance gain, but may result in problems
(crashes, non-termination) if the inputs do contain infinities or NaNs.
Default: True
Returns
-------
code : ndarray
code[i] gives the label of the ith obversation, that its code is
code_book[code[i]].
mind_dist : ndarray
min_dist[i] gives the distance between the ith observation and its
corresponding code.
Notes
-----
This could be faster when number of codebooks is small, but it
becomes a real memory hog when codebook is large. It requires
N by M by O storage where N=number of obs, M = number of
features, and O = number of codes.
"""
obs = _asarray_validated(obs, check_finite=check_finite)
code_book = _asarray_validated(code_book, check_finite=check_finite)
d = shape(obs)[1]
# code books and observations should have same number of features
if not d == code_book.shape[1]:
raise ValueError("""
code book(%d) and obs(%d) should have the same
number of features (eg columns)""" % (code_book.shape[1], d))
diff = obs[newaxis, :, :] - code_book[:, newaxis, :]
dist = sqrt(np.sum(diff * diff, -1))
code = argmin(dist, 0)
min_dist = minimum.reduce(dist, 0)
# The next line I think is equivalent and should be faster than the one
# above, but in practice didn't seem to make much difference:
# min_dist = choose(code,dist)
return code, min_dist
def headerize(image):
"""
This method generates the FITS headers for the final median and mean
images that are saved.
@type image: pyfits image
@param image: The FITS image of the stacked results
@returns: The pyfits image with the proper header values included
"""
# These values are used due to a bug in the astlib scale, so my own scale
# bzero and bscale values must be internally calculated.
scalingfactor = 0.000001
maxintvalue = 4294967294
# This adds the appropriate fits header values
image[0].header.update("STACKMIN", Stack.percentile_centralpix(0), "Minimum value of central pixel")
image[0].header.update("STACKMAX", Stack.percentile_centralpix(100), "Maximum value of central pixel")
image[0].header.update("STACK50", Stack.percentile_centralpix(50), "Median value of central pixel")
image[0].header.update("STACK25", Stack.percentile_centralpix(25), "25th Percentile value of central pixel")
image[0].header.update("STACK75", Stack.percentile_centralpix(75), "75th percentile value of central pixel")
image[0].header.update("STACKNUM", Stack.data.shape[2], "Number of accepted sources used")
image[0].header.update("BUNIT", units, "")
image[0].header.update("CRPIX1", 1, "")
image[0].header.update("CRPIX2", 1, "")
image[0].header.update("CTYPE1", "RA-CAR", "")
image[0].header.update("CTYPE2", "DEC-CAR", "")
# CDELT SHOULD be read from input fits file
image[0].header.update("CDELT1", cdelt1, "")
image[0].header.update("CDELT2", cdelt2, "")
image[0].header.update("CRVAL1", -((image[0].data.shape[0]) / 2.0) * image[0].header["CDELT1"], "")
image[0].header.update("CRVAL2", -((image[0].data.shape[1]) / 2.0) * image[0].header["CDELT2"], "")
# These lines are used to scale the image, as some software can only handle
# int values for the data array
min = minimum.reduce(minimum.reduce(image[0].data))
max = maximum.reduce(maximum.reduce(image[0].data))
# These lines are needed to prevent the lowest intensity pixels from being
# blanks
min = min - (scalingfactor * abs(min))
max = max + (scalingfactor * abs(max))
_zero = (max + min) / 2.0
_scale = (max - min) / (4294967294)
image[0].scale("int32", "", bzero=_zero, bscale=_scale)
return image
def __init__(self, data):
try:
max_str_len = max(len(str(maximum.reduce(data))),
len(str(minimum.reduce(data))))
self.format = '%' + str(max_str_len) + 'd'
except (TypeError, NotImplementedError):
# if reduce(data) fails, this instance will not be called, just
# instantiated in formatdict.
pass
except ValueError:
# this occurs when everything is NA
pass
def fillFormat(self, data):
import numpy.core.numeric as _nc
errstate = _nc.seterr(all='ignore')
try:
special = isnan(data) | isinf(data)
valid = not_equal(data, 0) & ~special
non_zero = absolute(data.compress(valid))
if len(non_zero) == 0:
max_val = 0.
min_val = 0.
else:
max_val = maximum.reduce(non_zero)
min_val = minimum.reduce(non_zero)
if max_val >= 1.e8:
self.exp_format = True
if not self.suppress_small and (min_val < 0.0001
or max_val / min_val > 1000.):
self.exp_format = True
finally:
_nc.seterr(**errstate)
if self.exp_format:
self.large_exponent = 0 < min_val < 1e-99 or max_val >= 1e100
self.max_str_len = 8 + self.precision
if self.large_exponent:
self.max_str_len += 1
if self.sign:
format = '%+'
else:
format = '%'
format = format + '%d.%de' % (self.max_str_len, self.precision)
else:
format = '%%.%df' % (self.precision,)
if len(non_zero):
precision = max([_digits(x, self.precision, format)
for x in non_zero])
else:
precision = 0
precision = min(self.precision, precision)
self.max_str_len = len(str(int(max_val))) + precision + 2
if _nc.any(special):
self.max_str_len = max(self.max_str_len,
len(_nan_str),
len(_inf_str) + 1)
if self.sign:
format = '%#+'
else:
format = '%#'
format = format + '%d.%df' % (self.max_str_len, precision)
self.special_fmt = '%%%ds' % (self.max_str_len,)
self.format = format
def py_vq2(obs, code_book):
"""2nd Python version of vq algorithm.
The algorithm simply computes the euclidian distance between each
observation and every frame in the code_book/
Parameters
----------
obs : ndarray
Expect a rank 2 array. Each row is one observation.
code_book : ndarray
Code book to use. Same format than obs. Should have same number of
features (eg columns) than obs.
Returns
-------
code : ndarray
code[i] gives the label of the ith obversation, that its code is
code_book[code[i]].
mind_dist : ndarray
min_dist[i] gives the distance between the ith observation and its
corresponding code.
Notes
-----
This could be faster when number of codebooks is small, but it
becomes a real memory hog when codebook is large. It requires
N by M by O storage where N=number of obs, M = number of
features, and O = number of codes.
"""
d = shape(obs)[1]
# code books and observations should have same number of features
if not d == code_book.shape[1]:
raise ValueError(
"""
code book(%d) and obs(%d) should have the same
number of features (eg columns)"""
% (code_book.shape[1], d)
)
diff = obs[newaxis, :, :] - code_book[:, newaxis, :]
dist = sqrt(np.sum(diff * diff, -1))
code = argmin(dist, 0)
min_dist = minimum.reduce(dist, 0) # the next line I think is equivalent
# - and should be faster
# min_dist = choose(code,dist) # but in practice, didn't seem to make
# much difference.
return code, min_dist
def plane(self, matrix):
from numpy import ravel, minimum, maximum, add, multiply, array, float32
matrix_1d = matrix.ravel()
dmin = minimum.reduce(matrix_1d)
if self.min == None or dmin < self.min:
self.min = dmin
dmax = maximum.reduce(matrix_1d)
if self.max == None or dmax > self.max:
self.max = dmax
self.sum += add.reduce(matrix_1d)
# TODO: Don't copy array to get standard deviation.
# Avoid overflow when squaring integral types
m2 = array(matrix_1d, float32)
multiply(m2, m2, m2)
self.sum2 += add.reduce(m2)
def plane(self, matrix):
from numpy import ravel, minimum, maximum, add, multiply, array, float32
matrix_1d = matrix.ravel()
dmin = minimum.reduce(matrix_1d)
if self.min == None or dmin < self.min:
self.min = dmin
dmax = maximum.reduce(matrix_1d)
if self.max == None or dmax > self.max:
self.max = dmax
self.sum += add.reduce(matrix_1d)
# TODO: Don't copy array to get standard deviation.
# Avoid overflow when squaring integral types
m2 = array(matrix_1d, float32)
multiply(m2, m2, m2)
self.sum2 += add.reduce(m2)
def py_vq2(obs, code_book):
"""2nd Python version of vq algorithm.
The algorithm simply computes the euclidian distance between each
observation and every frame in the code_book/
Parameters
----------
obs : ndarray
Expect a rank 2 array. Each row is one observation.
code_book : ndarray
Code book to use. Same format than obs. Should have same number of
features (eg columns) than obs.
Returns
-------
code : ndarray
code[i] gives the label of the ith obversation, that its code is
code_book[code[i]].
mind_dist : ndarray
min_dist[i] gives the distance between the ith observation and its
corresponding code.
Notes
-----
This could be faster when number of codebooks is small, but it
becomes a real memory hog when codebook is large. It requires
N by M by O storage where N=number of obs, M = number of
features, and O = number of codes.
"""
d = shape(obs)[1]
# code books and observations should have same number of features
if not d == code_book.shape[1]:
raise ValueError("""
code book(%d) and obs(%d) should have the same
number of features (eg columns)""" % (code_book.shape[1], d))
diff = obs[newaxis, :, :] - code_book[:, newaxis, :]
dist = sqrt(np.sum(diff * diff, -1))
code = argmin(dist, 0)
min_dist = minimum.reduce(dist, 0) #the next line I think is equivalent
# - and should be faster
#min_dist = choose(code,dist) # but in practice, didn't seem to make
# much difference.
return code, min_dist
def peaks():
vector, label = weeklydataset_sg_ndata(
"/media/4AC0AB31C0AB21E5/Documents and Settings/Claudio/Documenti/Thesis/Workloads/MSClaudio/ews/access_log-20110805.csv",
[],
)
x, target = aggregatebymins_sg_ndata(vector[1])
data = array(target)
t = array(x)
data = data.ravel()
length = len(data)
print length
step = 40
if length % step == 0:
data.shape = (length / step, step)
else:
data.resize((length / step, step))
max_data = maximum.reduce(data, 1)
min_data = minimum.reduce(data, 1)
pylab.plot(t, array(target), "b", label="signal")
return concatenate((max_data[:, newaxis], min_data[:, newaxis]), 1)
def __init__(self, data):
if data.dtype.kind == 'm':
v = data.view('i8')
max_str_len = max(len(str(maximum.reduce(v))),
len(str(minimum.reduce(v))))
self.format = '%' + str(max_str_len) + 'd'
def auto_interval ( data_low, data_high, max_ticks=9 ):
""" Calculates the tick interval for a range.
The boundaries for the data to be plotted on the axis are::
data_bounds = (data_low,data_high)
The function chooses the number of tick marks, which can be between
3 and max_ticks marks (including end points), and chooses tick intervals at
1, 2, 2.5, 5, 10, 20, ...
Returns
-------
interval : float
tick mark interval for axis
"""
range = float( data_high ) - float( data_low )
# We'll choose from between 2 and 8 tick marks.
# Preference is given to more ticks:
# Note reverse order and see kludge below...
divisions = arange( max_ticks-1, 2.0, -1.0 ) # for max_ticks=9, ( 7, 6, ..., 3 )
# Calculate the intervals for the divisions:
candidate_intervals = range / divisions
# Get magnitudes and mantissas for each candidate:
magnitudes = 10.0 ** floor( log10( candidate_intervals ) )
mantissas = candidate_intervals / magnitudes
# List of "pleasing" intervals between ticks on graph.
# Only the first magnitude are listed, higher mags others are inferred:
magic_intervals = array( ( 1.0, 2.0, 2.5, 5.0, 10.0 ) )
# Calculate the absolute differences between the candidates
# (with magnitude removed) and the magic intervals:
differences = abs( magic_intervals[:,newaxis] - mantissas )
# Find the division and magic interval combo that produce the
# smallest differences:
# KLUDGE: 'argsort' doesn't preserve the order of equal values,
# so we subtract a small, index dependent amount from each difference
# to force correct ordering.
sh = shape( differences )
small = 2.2e-16 * arange( sh[1] ) * arange( sh[0] )[:,newaxis]
small = small[::-1,::-1] #reverse the order
differences = differences - small
# ? Numeric should allow keyword "axis" ? comment out for now
#best_mantissa = minimum.reduce(differences,axis=0)
#best_magic = minimum.reduce(differences,axis=-1)
best_mantissa = minimum.reduce( differences, 0 )
best_magic = minimum.reduce( differences, -1 )
magic_index = argsort( best_magic )[0]
mantissa_index = argsort( best_mantissa )[0]
# The best interval is the magic_interval multiplied by the magnitude
# of the best mantissa:
interval = magic_intervals[ magic_index ]
magnitude = magnitudes[ mantissa_index ]
result = interval * magnitude
if result == 0.0:
result = finfo(float).eps
return result
def get_rgb(source, bands=None, **kwargs):
'''Extract RGB data for display from a SpyFile object or numpy array.
USAGE: rgb = get_rgb(source [, bands] [stretch=True]
[stretch_all=False] [bounds = (lower, upper)] )
Arguments:
`source` (:class:`spectral.SpyFile` or :class:`numpy.ndarray`):
Data source from which to extract the RGB data.
`bands` (list of `int`) (optional):
Optional triplet of indices which specifies the bands to extract
for the red, green, and blue components, respectively. If this
arg is not given, SpyFile object, it's metadata dict will be
checked to see if it contains a "default bands" item. If it does
not, then first, middle and last band will be returned.
Keyword Arguments:
`stretch` (bool, default True):
If the `stretch` keyword is True, the RGB values will be scaled
so the maximum value in the returned array will be 1.
`stretch_all` (bool, default False):
If this keyword is True, each color channel will be scaled
separately such that its maximum value is 1.
`bounds` (2-tuple of scalars):
If `bounds` is specified, the data will be scaled so that `lower`
and `upper` correspond to 0 and 1, respectively. Any values outside
of the range (`lower`, `upper`) will be clipped.
'''
from numpy import (take, zeros, repeat, ravel, minimum, maximum, clip,
float, int, newaxis)
from spectral.spectral import Image
from exceptions import TypeError
if not bands:
bands = []
if len(bands) != 0 and len(bands) != 1 and len(bands) != 3:
raise Exception("Invalid number of bands specified.")
monochrome = 0
if isinstance(source, Image) and len(source.shape) == 3:
# Figure out which bands to display
if len(bands) == 0:
# No bands specified. What should we show?
if hasattr(source, 'metadata') and \
'default bands' in source.metadata:
try:
bands = [int(b) for b in source.metadata['default bands']]
except:
pass
elif source.shape[-1] == 1:
bands = [0]
if len(bands) == 0:
# Pick the first, middle, and last bands
n = source.shape[-1]
bands = [0, n / 2, n - 1]
rgb = source.read_bands(bands).astype(float)
else:
# It should be a numpy array
s = source.shape
if len(s) == 2:
rgb = source[:, :, newaxis]
elif (len(s) == 3 and s[2] == 1):
rgb = source
elif len(s) == 3:
if s[2] == 3:
if len(bands) == 0:
# keep data as is.
rgb = source.astype(float)
elif len(bands) == 3:
if bands[0] == 0 and bands[1] == 1 and bands[2] == 2:
# Same as first 'if', bands just explicit.
rgb = source.astype(float)
else:
rgb = take(source, bands, 2).astype(float)
elif s[2] > 3 and (len(bands) == 1 or len(bands) == 3):
rgb = take(source, bands, 2).astype(float)
else:
rgb = take(source, [0, s[2] / 2, s[2] - 1], 2).astype(float)
else:
raise Exception('Invalid array shape for image display')
if 'colorScale' in kwargs:
color_scale = kwargs['colorScale']
warn('Keyword "colorScale" is deprecated. Use "color_scale"',
UserWarning)
else:
color_scale = kwargs.get('color_scale', None)
if 'autoScale' in kwargs:
auto_scale = kwargs['autoScale']
warn('Keyword "autoScale" is deprecated. Use "auto_scale"',
UserWarning)
else:
auto_scale = kwargs.get('auto_scale', False)
# If it's either color-indexed or monochrome
if rgb.shape[2] == 1:
s = rgb.shape
if "colors" in kwargs:
rgb = rgb.astype(int)
pal = kwargs["colors"]
rgb = pal[rgb[:,:,0]]
elif color_scale is not None:
# Colors should be generated from the supplied color scale
# This section assumes rgb colors in the range 0-255.
rgb = rgb[:, :, 0]
scale = color_scale
if auto_scale:
scale.set_range(min(rgb.ravel()), max(rgb.ravel()))
rgb3 = zeros((s[0], s[1], 3), int)
for i in range(s[0]):
for j in range(s[1]):
rgb3[i, j] = scale(rgb[i, j])
rgb = rgb3.astype(float) / 255.
else:
monochrome = 1
rgb = repeat(rgb, 3, 2).astype(float)
if "colors" not in kwargs:
# Perform any requested color enhancements.
if "stretch" in kwargs or "bounds" not in kwargs:
stretch = 1
if "bounds" in kwargs:
# Stretch each color within the value bounds
(lower, upper) = kwargs["bounds"]
rgb = (rgb - lower) / (upper - lower)
rgb = clip(rgb, 0, 1)
elif kwargs.get("stretch_all", False):
# Stretch each color over its full range
for i in range(rgb.shape[2]):
mmin = minimum.reduce(ravel(rgb[:, :, i]))
mmax = maximum.reduce(ravel(rgb[:, :, i]))
rgb[:, :, i] = (rgb[:, :, i] - mmin) / (mmax - mmin)
elif stretch or (kwargs.get("stretch_all", False) and monochrome):
# Stretch so highest color channel value is 1
mmin = minimum.reduce(ravel(rgb))
mmax = maximum.reduce(ravel(rgb))
rgb = (rgb - mmin) / (mmax - mmin)
return rgb
def auto_interval(data_low, data_high, max_ticks=9):
""" Calculates the tick interval for a range.
The boundaries for the data to be plotted on the axis are::
data_bounds = (data_low,data_high)
The function chooses the number of tick marks, which can be between
3 and max_ticks marks (including end points), and chooses tick intervals at
1, 2, 2.5, 5, 10, 20, ...
Returns
-------
interval : float
tick mark interval for axis
"""
range = float(data_high) - float(data_low)
# We'll choose from between 2 and 8 tick marks.
# Preference is given to more ticks:
# Note reverse order and see kludge below...
divisions = arange(max_ticks - 1, 2.0,
-1.0) # for max_ticks=9, ( 7, 6, ..., 3 )
# Calculate the intervals for the divisions:
candidate_intervals = range / divisions
# Get magnitudes and mantissas for each candidate:
magnitudes = 10.0**floor(log10(candidate_intervals))
mantissas = candidate_intervals / magnitudes
# List of "pleasing" intervals between ticks on graph.
# Only the first magnitude are listed, higher mags others are inferred:
magic_intervals = array((1.0, 2.0, 2.5, 5.0, 10.0))
# Calculate the absolute differences between the candidates
# (with magnitude removed) and the magic intervals:
differences = abs(magic_intervals[:, newaxis] - mantissas)
# Find the division and magic interval combo that produce the
# smallest differences:
# KLUDGE: 'argsort' doesn't preserve the order of equal values,
# so we subtract a small, index dependent amount from each difference
# to force correct ordering.
sh = shape(differences)
small = 2.2e-16 * arange(sh[1]) * arange(sh[0])[:, newaxis]
small = small[::-1, ::-1] #reverse the order
differences = differences - small
# ? Numeric should allow keyword "axis" ? comment out for now
#best_mantissa = minimum.reduce(differences,axis=0)
#best_magic = minimum.reduce(differences,axis=-1)
best_mantissa = minimum.reduce(differences, 0)
best_magic = minimum.reduce(differences, -1)
magic_index = argsort(best_magic)[0]
mantissa_index = argsort(best_mantissa)[0]
# The best interval is the magic_interval multiplied by the magnitude
# of the best mantissa:
interval = magic_intervals[magic_index]
magnitude = magnitudes[mantissa_index]
result = interval * magnitude
if result == 0.0:
result = finfo(float).eps
return result
