def __call__(self, population):
'''
Selects the population.
:Parameters:
population
The list of chromosomes that should be operated over. The given list
is modified, so be aware that the old generation will not be
available after stepping the GA.
:Returns:
The new population.
'''
facc = add.accumulate(population.fitness)
newp = [ ]
cs = uniform(0., 1.)
si = 0
l = len(population)
l1 = 1. / l
for j in xrange(l):
while facc[si] < cs:
si = (si + 1) % l
newp.append(Chromosome(population[si]))
cs = (cs + l1) % 1
population[:] = newp
return population
def pg(t_start, t_stop, rate):
#set-up random number generator
rng = random.RandomState()
rng.seed
#Calc expected number of events give PP rate and duration
n_exp = ceil((t_stop - t_start) * rate)
iets = rng.exponential(1.0 / rate, n_exp)
spikes = t_start + add.accumulate(iets)
i = searchsorted(spikes, t_stop)
#Add extra spikes if length too short
extra_spikes = []
if i == len(spikes):
# ISI buf overrun
t_last = spikes[-1] + rng.exponential(1.0 / rate, 1)[0]
while (t_last < t_stop):
extra_spikes.append(t_last)
t_last += rng.exponential(1.0 / rate, 1)[0]
spikes = concatenate((spikes, extra_spikes))
#print "ISI buf overrun handled. len(spikes)=%d, len(extra_spikes)=%d" % (len(spikes),len(extra_spikes))
else:
spikes = resize(spikes, (i, ))
#print "len(spikes)=%d" % (len(spikes))
return spikes
def pg(t_start, t_stop, rate):
#set-up random number generator
rng = random.RandomState()
rng.seed
#Calc expected number of events give PP rate and duration
n_exp = ceil((t_stop-t_start)*rate)
iets = rng.exponential(1.0/rate, n_exp)
spikes = t_start + add.accumulate(iets)
i = searchsorted(spikes, t_stop)
#Add extra spikes if length too short
extra_spikes = []
if i==len(spikes):
# ISI buf overrun
t_last = spikes[-1] + rng.exponential(1.0/rate, 1)[0]
while (t_last<t_stop):
extra_spikes.append(t_last)
t_last += rng.exponential(1.0/rate, 1)[0]
spikes = concatenate((spikes,extra_spikes))
#print "ISI buf overrun handled. len(spikes)=%d, len(extra_spikes)=%d" % (len(spikes),len(extra_spikes))
else:
spikes = resize(spikes,(i,))
#print "len(spikes)=%d" % (len(spikes))
return spikes
def __call__(self, population):
'''
Selects the population.
:Parameters:
population
The list of chromosomes that should be operated over. The given list
is modified, so be aware that the old generation will not be
available after stepping the GA.
:Returns:
The new population.
'''
facc = add.accumulate(population.fitness)
newp = [ ]
l = len(population)
for j in xrange(l):
m = randrange(l)
n = randrange(l)
if facc[m] > facc[n]:
newp.append(Chromosome(population[m]))
else:
newp.append(Chromosome(population[n]))
population[:] = newp
return population
def cumtrapz(y, x=None, dx=1.0, axis=-1):
"""Cumulatively integrate y(x) using samples along the given axis
and the composite trapezoidal rule. If x is None, spacing given by dx
is assumed.
See also:
quad - adaptive quadrature using QUADPACK
romberg - adaptive Romberg quadrature
quadrature - adaptive Gaussian quadrature
fixed_quad - fixed-order Gaussian quadrature
dblquad, tplquad - double and triple integrals
romb, trapz - integrators for sampled data
cumtrapz - cumulative integration for sampled data
ode, odeint - ODE integrators
"""
y = asarray(y)
if x is None:
d = dx
else:
d = diff(x, axis=axis)
nd = len(y.shape)
slice1 = tupleset((slice(None), ) * nd, axis, slice(1, None))
slice2 = tupleset((slice(None), ) * nd, axis, slice(None, -1))
return add.accumulate(d * (y[slice1] + y[slice2]) / 2.0, axis)
def _compute(self):
# tvs = [x.getResult() for x in self._children]
# tvStarts = [x.startsAsNumpyArray()for x in tvs]
# tvEnds = [x.endsAsNumpyArray() for x in tvs]
# numTracks = len(tvStarts)
#
# tvCodedStarts = []
# tvCodedEnds = []
# for i in xrange(numTracks):
# tvCodedStarts.append(tvStarts[i] * 4 + 3)
# tvCodedEnds.append(tvEnds[i] * 4 + 1)
#
# allSortedCodedEvents = concatenate((concatenate(tvCodedStarts), concatenate(tvCodedEnds) ))
# allSortedCodedEvents.sort()
allSortedCodedEvents = self._children[0].getResult()
allEventCodes = (allSortedCodedEvents % 4) - 2
allSortedDecodedEvents = allSortedCodedEvents / 4
allEventLengths = allSortedDecodedEvents[
1:] - allSortedDecodedEvents[:-1]
cumulativeCoverStatus = add.accumulate(allEventCodes)
return allSortedDecodedEvents, allEventLengths, cumulativeCoverStatus
def choose_initial_pp(data, k, distfunc):
"""
Choose randomly k different centroids using the kmeans++ heuristic
by David Arthur and Sergei Vassilvitskii (see the article "k-means++:
The Advantages of Careful Seeding".
:data The elements being clustered
:k The number of clusters
:distfunc Function to calculate the distance between two elements.
"""
from bisect import bisect
from numpy import add
# Calculate squared distance
distance2 = lambda c, x: distfunc(c, x)**2
# The first centroid is a random one
centroids = [random.choice(data)]
tries = 0
while len(centroids) < k and tries < (k * 5):
mindists = [min((distance2(c, x), x) for c in centroids)
for x in data if x not in centroids]
# Divide because we add it twice: first for (c, x) and then for (x, c)
totaldist = float(sum(d for d, _ in mindists))
probs = [(d / totaldist) for d, _ in mindists]
pos = bisect(add.accumulate(probs), random.random())
centroids.append(mindists[pos - 1][1])
tries += 1
if len(centroids) < k:
centroids.extend(random.sample(data, k - len(centroids)))
return centroids
def cumtrapz(y, x=None, dx=1.0, axis=-1):
"""Cumulatively integrate y(x) using samples along the given axis
and the composite trapezoidal rule. If x is None, spacing given by dx
is assumed.
See also:
quad - adaptive quadrature using QUADPACK
romberg - adaptive Romberg quadrature
quadrature - adaptive Gaussian quadrature
fixed_quad - fixed-order Gaussian quadrature
dblquad, tplquad - double and triple integrals
romb, trapz - integrators for sampled data
cumtrapz - cumulative integration for sampled data
ode, odeint - ODE integrators
"""
y = asarray(y)
if x is None:
d = dx
else:
d = diff(x, axis=axis)
nd = len(y.shape)
slice1 = tupleset((slice(None),) * nd, axis, slice(1, None))
slice2 = tupleset((slice(None),) * nd, axis, slice(None, -1))
return add.accumulate(d * (y[slice1] + y[slice2]) / 2.0, axis)
def construct(self, iconMapping):
matrix = iconMapping.matrix
max_n = matrix.max()
numberOfCellsHit = zeros(max_n + 1)
for a in matrix:
numberOfCellsHit[a] = numberOfCellsHit[a] + 1
self.totalNumberOfCellsHit = add.accumulate(numberOfCellsHit)
self.maxTotalNumberOfCellsHit = self.totalNumberOfCellsHit.max()
image = Image.new("RGB", (matrix.shape[0], matrix.shape[1]))
for ny in range(matrix.shape[0]):
for nx in range(matrix.shape[1]):
image.putpixel((nx, ny), self.color(matrix[nx, ny]))
image.save(self.fileName)
def weighted_median(x, w):
"""
Calculates the weighted median, that is the minimizer of
argmin {\sum w_i |x_i - \mu|}
@param x: input array
@param w: array of weights
"""
from numpy import sum, add, argsort, sort
w = w / w.sum()
w = w[argsort(x)]
x = sort(x)
j = sum(add.accumulate(w) < 0.5)
return x[j]
def weighted_median(x, w):
"""
Calculates the weighted median, that is the minimizer of
argmin {\sum w_i |x_i - \mu|}
@param x: input array
@param w: array of weights
"""
from numpy import sum, add, argsort, sort
w = w / w.sum()
w = w[argsort(x)]
x = sort(x)
j = sum(add.accumulate(w) < 0.5)
return x[j]
def cumtrapz(y, x=None, dx=1.0, axis=-1):
"""
Cumulatively integrate y(x) using samples along the given axis
and the composite trapezoidal rule. If x is None, spacing given by dx
is assumed.
Parameters
----------
y : array
x : array, optional
dx : int, optional
axis : int, optional
Specifies the axis to cumulate:
- -1 --> X axis
- 0 --> Z axis
- 1 --> Y axis
See Also
--------
quad: adaptive quadrature using QUADPACK
romberg: adaptive Romberg quadrature
quadrature: adaptive Gaussian quadrature
fixed_quad: fixed-order Gaussian quadrature
dblquad: double integrals
tplquad: triple integrals
romb: integrators for sampled data
trapz: integrators for sampled data
cumtrapz: cumulative integration for sampled data
ode: ODE integrators
odeint: ODE integrators
"""
y = asarray(y)
if x is None:
d = dx
else:
d = diff(x, axis=axis)
nd = len(y.shape)
slice1 = tupleset((slice(None),) * nd, axis, slice(1, None))
slice2 = tupleset((slice(None),) * nd, axis, slice(None, -1))
return add.accumulate(d * (y[slice1] + y[slice2]) / 2.0, axis)
def cumtrapz(y, x=None, dx=1.0, axis=-1):
"""
Cumulatively integrate y(x) using samples along the given axis
and the composite trapezoidal rule. If x is None, spacing given by dx
is assumed.
Parameters
----------
y : array
x : array, optional
dx : int, optional
axis : int, optional
Specifies the axis to cumulate:
- -1 --> X axis
- 0 --> Z axis
- 1 --> Y axis
See Also
--------
quad: adaptive quadrature using QUADPACK
romberg: adaptive Romberg quadrature
quadrature: adaptive Gaussian quadrature
fixed_quad: fixed-order Gaussian quadrature
dblquad: double integrals
tplquad: triple integrals
romb: integrators for sampled data
trapz: integrators for sampled data
cumtrapz: cumulative integration for sampled data
ode: ODE integrators
odeint: ODE integrators
"""
y = asarray(y)
if x is None:
d = dx
else:
d = diff(x, axis=axis)
nd = len(y.shape)
slice1 = tupleset((slice(None), ) * nd, axis, slice(1, None))
slice2 = tupleset((slice(None), ) * nd, axis, slice(None, -1))
return add.accumulate(d * (y[slice1] + y[slice2]) / 2.0, axis)
def _compute(self):
tvs = [x.getResult() for x in self._children]
from numpy import array
# tvStartsOld = [x.startsAsNumpyArray()for x in tvs]
# tvEndsOld = [x.endsAsNumpyArray() for x in tvs]
tvStarts = [array(x.startsAsNumpyArray(), dtype='int64') for x in tvs]
tvEnds = [array(x.endsAsNumpyArray(), dtype='int64') for x in tvs]
numTracks = len(tvStarts)
assert numTracks < 34, 'Maximum supported nr. of tracks for this statistic is 33'
multiplier = 2**(numTracks + 1)
#assert no overlaps..
#create arrays multiplied by 8 to use last three bits to code event type,
#Last three bits: relative to 4 (100): +/- 1 for start/end of track1, +/- 2 for track2..
tvCodedStarts = []
tvCodedEnds = []
for i in xrange(numTracks):
tvCodedStarts.append(tvStarts[i] * multiplier + (2**numTracks) +
(2**i))
tvCodedEnds.append(tvEnds[i] * multiplier + (2**numTracks) -
(2**i))
# t1CodedStarts = t1s * 8 +5
# t1CodedEnds= t1e * 8 +3
# t2CodedStarts = t2s * 8 +6
# t2CodedEnds= t2e * 8 +2
allSortedCodedEvents = concatenate(
(concatenate(tvCodedStarts), concatenate(tvCodedEnds)))
allSortedCodedEvents.sort()
allEventCodes = (allSortedCodedEvents % multiplier) - (2**numTracks)
allSortedDecodedEvents = allSortedCodedEvents / multiplier
allEventLengths = allSortedDecodedEvents[
1:] - allSortedDecodedEvents[:-1]
#due to the coding, the last bit now has status of track1, and the second last bit status of track2
#thus, 3 is cover by both, 2 is cover by only track2, 1 is cover by only track1, 0 is no cover
#this works as there are no overlaps, and bits will thus not "spill over"..
cumulativeCoverStatus = add.accumulate(allEventCodes)
return allSortedDecodedEvents, allEventLengths, cumulativeCoverStatus
def sample_from_histogram(p, n_samples=1):
"""
returns the indice of bin according to the histogram p
@param p: histogram
@type p: numpy.array
@param n_samples: number of samples to generate
@type n_samples: integer
"""
from numpy import add, less, argsort, take, arange
from numpy.random import random
indices = argsort(p)
indices = take(indices, arange(len(p) - 1, -1, -1))
c = add.accumulate(take(p, indices)) / add.reduce(p)
return indices[add.reduce(less.outer(c, random(n_samples)), 0)]
def get_bounds_from_histogram(hist,bin_edges,confidence=0.95):
"""Return the bins corresponding to upper and lower confidence limits
hist -- an array of histogram values
bin_edges -- an array of bin edges (length equal to hist + 1)
confidence -- the percent confidence required. For example 0.95
will result in the 5% and 95% confidence limits being returned
NOTE: since data are binned, the 5% or 95% confidence intervals
will not be exact. We choose to return the more conservative
bin, so actual confidence will be <= 5% or >= 95%
"""
total = sum(hist)
normed_hist = hist/total
#print "normed_hist:",normed_hist
#The frequency of draws allowed to fall on *each* side
#of the confidence interval is half the total
confidence_each_tail = (1.0 - confidence)/2.0
cum_sum = add.accumulate(normed_hist)
#print "cum_sum:",cum_sum
upper_indices = where(cum_sum > 1.0-confidence_each_tail)
lower_indices = where(cum_sum < confidence_each_tail)
#print "lower_indices:",lower_indices
#print "upper_indices:",upper_indices
lower_index = amax(lower_indices)
upper_index = amin(upper_indices)
lower_bin_index = lower_index + 1
upper_bin_index = upper_index + 1
upper_bound = bin_edges[upper_bin_index]
lower_bound = bin_edges[lower_bin_index]
return lower_bound,upper_bound
def test_accumulate(self):
from numpy import add, subtract, multiply, divide, arange, dtype
assert (add.accumulate([2, 3, 5]) == [2, 5, 10]).all()
assert (multiply.accumulate([2, 3, 5]) == [2, 6, 30]).all()
a = arange(4).reshape(2,2)
b = add.accumulate(a, axis=0)
assert (b == [[0, 1], [2, 4]]).all()
b = add.accumulate(a, 1)
assert (b == [[0, 1], [2, 5]]).all()
b = add.accumulate(a) #default axis is 0
assert (b == [[0, 1], [2, 4]]).all()
# dtype
a = arange(0, 3, 0.5).reshape(2, 3)
b = add.accumulate(a, dtype=int, axis=1)
print b
assert (b == [[0, 0, 1], [1, 3, 5]]).all()
assert b.dtype == int
assert add.accumulate([True]*200)[-1] == 200
assert add.accumulate([True]*200).dtype == dtype('int')
assert subtract.accumulate([True]*200).dtype == dtype('bool')
assert divide.accumulate([True]*200).dtype == dtype('int8')
def sample(self):
from numpy.random import random
from numpy import add
from csb.numeric import log_sum_exp
log_m = self.log_masses()
log_M = log_sum_exp(log_m)
c = add.accumulate(exp(log_m - log_M))
u = random()
j = (u > c).sum()
a = self.dh[j]
z = self.z()
xmin, xmax = z[j], z[j + 1]
u = random()
if a > 0:
return xmax + log(u + (1 - u) * exp(-a * (xmax - xmin))) / a
else:
return xmin + log(u + (1 - u) * exp(a * (xmax - xmin))) / a
def sample(self):
from numpy.random import random
from numpy import add
from csb.numeric import log_sum_exp
log_m = self.log_masses()
log_M = log_sum_exp(log_m)
c = add.accumulate(exp(log_m - log_M))
u = random()
j = (u > c).sum()
a = self.dh[j]
z = self.z()
xmin, xmax = z[j], z[j + 1]
u = random()
if a > 0:
return xmax + log(u + (1 - u) * exp(-a * (xmax - xmin))) / a
else:
return xmin + log(u + (1 - u) * exp(a * (xmax - xmin))) / a
def choose_initial_pp(data, k, distfunc):
"""
Choose randomly k different centroids using the kmeans++ heuristic
by David Arthur and Sergei Vassilvitskii (see the article "k-means++:
The Advantages of Careful Seeding".
:data The elements being clustered
:k The number of clusters
:distfunc Function to calculate the distance between two elements.
"""
from bisect import bisect
from numpy import add
# Calculate squared distance
distance2 = lambda c, x: distfunc(c, x)**2
# The first centroid is a random one
centroids = [random.choice(data)]
tries = 0
while len(centroids) < k and tries < (k * 5):
mindists = [
min((distance2(c, x), x) for c in centroids) for x in data
if x not in centroids
]
# Divide because we add it twice: first for (c, x) and then for (x, c)
totaldist = float(sum(d for d, _ in mindists))
probs = [(d / totaldist) for d, _ in mindists]
pos = bisect(add.accumulate(probs), random.random())
centroids.append(mindists[pos - 1][1])
tries += 1
if len(centroids) < k:
centroids.extend(random.sample(data, k - len(centroids)))
return centroids
def cumtrapz(y, x=None, dx=1.0, axis=-1, initial=None):
"""
Cumulatively integrate y(x) using the composite trapezoidal rule.
Parameters
----------
y : array_like
Values to integrate.
x : array_like, optional
The coordinate to integrate along. If None (default), use spacing `dx`
between consecutive elements in `y`.
dx : int, optional
Spacing between elements of `y`. Only used if `x` is None.
axis : int, optional
Specifies the axis to cumulate. Default is -1 (last axis).
initial : scalar, optional
If given, uses this value as the first value in the returned result.
Typically this value should be 0. Default is None, which means no
value at ``x[0]`` is returned and `res` has one element less than `y`
along the axis of integration.
Returns
-------
res : ndarray
The result of cumulative integration of `y` along `axis`.
If `initial` is None, the shape is such that the axis of integration
has one less value than `y`. If `initial` is given, the shape is equal
to that of `y`.
See Also
--------
numpy.cumsum, numpy.cumprod
quad: adaptive quadrature using QUADPACK
romberg: adaptive Romberg quadrature
quadrature: adaptive Gaussian quadrature
fixed_quad: fixed-order Gaussian quadrature
dblquad: double integrals
tplquad: triple integrals
romb: integrators for sampled data
ode: ODE integrators
odeint: ODE integrators
Examples
--------
>>> from scipy import integrate
>>> x = np.linspace(-2, 2, num=20)
>>> y = x
>>> y_int = integrate.cumtrapz(y, x, initial=0)
>>> plt.plot(x, y_int, 'ro', x, y[0] + 0.5 * x**2, 'b-')
>>> plt.show()
"""
y = asarray(y)
if x is None:
d = dx
else:
d = diff(x, axis=axis)
nd = len(y.shape)
slice1 = tupleset((slice(None),)*nd, axis, slice(1, None))
slice2 = tupleset((slice(None),)*nd, axis, slice(None, -1))
res = add.accumulate(d * (y[slice1] + y[slice2]) / 2.0, axis)
if initial is not None:
if not np.isscalar(initial):
raise ValueError("`initial` parameter should be a scalar.")
shape = list(res.shape)
shape[axis] = 1
res = np.concatenate([np.ones(shape, dtype=res.dtype) * initial, res],
axis=axis)
return res
def load(self):
"""
Attempts to load candidate data from the file, performs file consistency checks if the
debug flag is set to true. Much of this code has been extracted from PRESTO by Scott Ransom.
Please see:
http://www.cv.nrao.edu/~sransom/presto/
https://github.com/scottransom/presto
Parameters:
N/A
Return:
N/A
"""
infile = open(self.cand, "rb")
# The code below appears to have been taken from Presto. So it maybe
# helpful to look at the Presto github repository (see above) to get a better feel
# for what this code is doing. I certainly have no idea what is going on. Although
# data is being unpacked in a specific order.
swapchar = '<' # this is little-endian
data = infile.read(5 * 4)
testswap = struct.unpack(swapchar + "i" * 5, data)
# This is a hack to try and test the endianness of the data.
# None of the 5 values should be a large positive number.
if (fabs(asarray(testswap))).max() > 100000:
swapchar = '>' # this is big-endian
(self.numdms, self.numperiods, self.numpdots, self.nsub,
self.npart) = struct.unpack(swapchar + "i" * 5, data)
(self.proflen, self.numchan, self.pstep, self.pdstep, self.dmstep,
self.ndmfact, self.npfact) = struct.unpack(swapchar + "i" * 7,
infile.read(7 * 4))
self.filenm = infile.read(
struct.unpack(swapchar + "i", infile.read(4))[0])
self.candnm = infile.read(
struct.unpack(swapchar + "i", infile.read(4))[0])
self.telescope = infile.read(
struct.unpack(swapchar + "i", infile.read(4))[0])
self.pgdev = infile.read(
struct.unpack(swapchar + "i", infile.read(4))[0])
test = infile.read(16)
has_posn = 1
for ii in range(16):
if test[ii] not in '0123456789:.-\0':
has_posn = 0
break
if has_posn:
self.rastr = test[:test.find('\0')]
test = infile.read(16)
self.decstr = test[:test.find('\0')]
(self.dt, self.startT) = struct.unpack(swapchar + "dd",
infile.read(2 * 8))
else:
self.rastr = "Unknown"
self.decstr = "Unknown"
(self.dt, self.startT) = struct.unpack(swapchar + "dd", test)
(self.endT, self.tepoch, self.bepoch, self.avgvoverc, self.lofreq,
self.chan_wid, self.bestdm) = struct.unpack(swapchar + "d" * 7,
infile.read(7 * 8))
(self.topo_pow, tmp) = struct.unpack(swapchar + "f" * 2,
infile.read(2 * 4))
(self.topo_p1, self.topo_p2,
self.topo_p3) = struct.unpack(swapchar + "d" * 3, infile.read(3 * 8))
(self.bary_pow, tmp) = struct.unpack(swapchar + "f" * 2,
infile.read(2 * 4))
(self.bary_p1, self.bary_p2,
self.bary_p3) = struct.unpack(swapchar + "d" * 3, infile.read(3 * 8))
(self.fold_pow, tmp) = struct.unpack(swapchar + "f" * 2,
infile.read(2 * 4))
(self.fold_p1, self.fold_p2,
self.fold_p3) = struct.unpack(swapchar + "d" * 3, infile.read(3 * 8))
(self.orb_p, self.orb_e, self.orb_x, self.orb_w, self.orb_t,
self.orb_pd, self.orb_wd) = struct.unpack(swapchar + "d" * 7,
infile.read(7 * 8))
self.dms = asarray(
struct.unpack(swapchar + "d" * self.numdms,
infile.read(self.numdms * 8)))
if self.numdms == 1:
self.dms = self.dms[0]
self.periods = asarray(
struct.unpack(swapchar + "d" * self.numperiods,
infile.read(self.numperiods * 8)))
self.pdots = asarray(
struct.unpack(swapchar + "d" * self.numpdots,
infile.read(self.numpdots * 8)))
self.numprofs = self.nsub * self.npart
if (swapchar == '<'): # little endian
self.profs = zeros((self.npart, self.nsub, self.proflen),
dtype='d')
for ii in range(self.npart):
for jj in range(self.nsub):
try:
self.profs[ii,
jj, :] = fromfile(infile, float64,
self.proflen)
except Exception: # Catch *all* exceptions.
pass
#print ""
else:
self.profs = asarray(
struct.unpack(swapchar + "d" * self.numprofs * self.proflen,
infile.read(self.numprofs * self.proflen * 8)))
self.profs = reshape(self.profs,
(self.npart, self.nsub, self.proflen))
self.binspersec = self.fold_p1 * self.proflen
self.chanpersub = self.numchan / self.nsub
self.subdeltafreq = self.chan_wid * self.chanpersub
self.hifreq = self.lofreq + (self.numchan - 1) * self.chan_wid
self.losubfreq = self.lofreq + self.subdeltafreq - self.chan_wid
self.subfreqs = arange(self.nsub,
dtype='d') * self.subdeltafreq + self.losubfreq
self.subdelays_bins = zeros(self.nsub, dtype='d')
self.killed_subbands = []
self.killed_intervals = []
self.pts_per_fold = []
# Note: a foldstats struct is read in as a group of 7 doubles
# the correspond to, in order:
# numdata, data_avg, data_var, numprof, prof_avg, prof_var, redchi
self.stats = zeros((self.npart, self.nsub, 7), dtype='d')
for ii in range(self.npart):
currentstats = self.stats[ii]
for jj in range(self.nsub):
if (swapchar == '<'): # little endian
try:
currentstats[jj] = fromfile(infile, float64, 7)
except Exception: # Catch *all* exceptions.
pass
#print ""
else:
try:
currentstats[jj] = asarray(
struct.unpack(swapchar + "d" * 7,
infile.read(7 * 8)))
except Exception: # Catch *all* exceptions.
pass
#print ""
self.pts_per_fold.append(
self.stats[ii][0][0]) # numdata from foldstats
self.start_secs = add.accumulate([0] +
self.pts_per_fold[:-1]) * self.dt
self.pts_per_fold = asarray(self.pts_per_fold)
self.mid_secs = self.start_secs + 0.5 * self.dt * self.pts_per_fold
if (not self.tepoch == 0.0):
self.start_topo_MJDs = self.start_secs / 86400.0 + self.tepoch
self.mid_topo_MJDs = self.mid_secs / 86400.0 + self.tepoch
if (not self.bepoch == 0.0):
self.start_bary_MJDs = self.start_secs / 86400.0 + self.bepoch
self.mid_bary_MJDs = self.mid_secs / 86400.0 + self.bepoch
self.Nfolded = add.reduce(self.pts_per_fold)
self.T = self.Nfolded * self.dt
self.avgprof = (self.profs / self.proflen).sum()
self.varprof = self.calc_varprof()
self.barysubfreqs = self.subfreqs
infile.close()
# If explicit debugging required.
if (self.debug):
# If candidate file is invalid in some way...
if (self.isValid() == False):
print "Invalid PFD candidate: ", self.cand
raise Exception(
"Invalid PFD candidate: PFDFile.py (Line 214).")
# Candidate file is valid.
else:
print "Candidate file valid."
self.profile = array(self.getprofile())
# Just go directly to feature generation without checks.
else:
self.out("Candidate validity checks skipped.", "")
self.profile = array(self.getprofile())
def __call__(self, points):
'''Return the global coordinates of the supplied local points.
'''
# number of local grid points for each coordinate direction
# NOTE: values must range between 0 and 1
#
xi_, yi_, zi_ = points[:, 0], points[:, 1], points[:, 2]
# insert imperfection (shift the middle node of the shell upwards)
imp = self.z_imperfection_factor
zi_ += imp * xi_ + imp * yi_ - 2 * imp * xi_ * yi_
# size of total structure
#
# @todo: move to class definition of "mushroof_model" and send to "__call__"
scale_size = self.scale_size
# @todo: add "_quarter" (see above)
length_xy_tot = self.length_xy_quarter * scale_size
n_elems_xy_quarter = self.n_elems_xy_quarter
# print 'HPShell n_elems_xy_quarter', n_elems_xy_quarter
# distance from origin for each mushroof_part
#
def d_origin_fn(self, coords):
if self.mushroof_part == 'quarter':
return coords
if self.mushroof_part == 'one':
return abs(2.0 * coords - 1.0)
# @todo: corresponding "scale_factor" needs to be added
# in order for this to work
if self.mushroof_part == 'four':
return where(coords < 0.5, abs(4 * coords - 1), abs(-4 * coords + 3))
# element at column shift
#
if self.shift_elems == True:
# define the origin for each model part
#
def origin_fn(self, coords):
if self.mushroof_part == 'quarter':
return zeros_like(coords)
if self.mushroof_part == 'one':
return ones_like(xi_) * 0.5
if self.mushroof_part == 'four':
return where(coords < 0.5, 0.25, 0.75)
def piecewise_linear_fn(x, x_fix_arr_, y_fix_arr_):
'''creates a piecewise linear_fn going through the fix_points
values need to be normed running between 0..1
and values have to be unique'''
x_fix_arr_ = hstack((0, x_fix_arr_, 1))
y_fix_arr_ = hstack((0, y_fix_arr_, 1))
rbf_fn_ = Rbf(x_fix_arr_, y_fix_arr_, function = 'linear') #rbf has to be linear
return rbf_fn_(x)
# define origin for quarter
#
xi_origin_arr_ = origin_fn(self, xi_)
yi_origin_arr_ = origin_fn(self, yi_)
# print 'xi_origin_arr_', xi_origin_arr_
# delta towards origin
#
xi_delta_arr_ = (xi_ - xi_origin_arr_) * scale_size
yi_delta_arr_ = (yi_ - yi_origin_arr_) * scale_size
# print 'xi_delta_arr_', xi_delta_arr
# define sign
#
xi_sign_arr = where(xi_delta_arr_ == 0., 0., xi_delta_arr_ / abs(xi_delta_arr_))
yi_sign_arr = where(yi_delta_arr_ == 0., 0., yi_delta_arr_ / abs(yi_delta_arr_))
# print 'xi_sign_arr', xi_sign_arr
# fix points defined in shift array as normelized values
#
x_fix_ = self.shift_array[:, 0] / self.length_xy_quarter
# print 'x_fix_', x_fix_
y_fix_ = self.shift_array[:, 1] / self.length_xy_quarter
n_fix_ = add.accumulate(self.shift_array[:, 2]) / n_elems_xy_quarter
# print 'add.accumulate( self.shift_array[:, 2] )', add.accumulate( self.shift_array[:, 2] )
# print 'n_fix_', n_fix_
# print 'piecewise_linear_fn', piecewise_linear_fn( abs( xi_delta_arr_ ),
# n_fix_,
# x_fix_ ) / scale_size
# new xi_
#
xi_ = xi_origin_arr_ + xi_sign_arr * piecewise_linear_fn(abs(xi_delta_arr_),
n_fix_,
x_fix_) / scale_size
# print 'xi_new', xi_
# new yi
#
yi_ = yi_origin_arr_ + yi_sign_arr * piecewise_linear_fn(abs(yi_delta_arr_),
n_fix_,
y_fix_) / scale_size
#-------------------------------------------------------------------------------------
# values are used to calculate the z-coordinate using RBF-function of the quarter
# (= values of the distance to the origin as absolute value)
#
xi_rbf_ = d_origin_fn(self, xi_)
# print 'xi_rbf_', xi_rbf_
yi_rbf_ = d_origin_fn(self, yi_)
# get the z-value at the supplied local grid points
# of the lower face
#
zi_lower_ = self.rbf_l_(xi_rbf_, yi_rbf_)
# get the z-value at the supplied local grid points
# of the upper face
#
zi_upper_ = self.rbf_u_(xi_rbf_, yi_rbf_)
# constant edge element transformation
#
if self.const_reinf_layer_elem == True:
# arrange and check data
#
if self.t_reinf_layer > self.t_shell / 2. or self.n_elems_z < 3:
print '--- constant edge element transformation canceled ---'
print 'the following condition needs to be fullfilled: \n'
print 'self.t_reinf_layer <= self.t_shell/2 and self.n_elems_z >= 3'
else:
n_elems_z = float(self.n_elems_z)
# normed thickness will evaluate as t_reinf_layer at each element
t_reinf_layer_ = self.t_reinf_layer / self.length_z / (zi_upper_ - zi_lower_)
# zi_old set off from top which needs to be shifted
delta_ = self.n_elems_reinf_layer / n_elems_z
# get upper, lower and internal coordinates, that need to be shifted
zi_lower = where(zi_ <= delta_)
zi_upper = where(abs(1 - zi_) <= delta_ + 1e-10)
zi_inter = where(abs(zi_ - 0.5) < 0.5 - (delta_ + 1e-10))
# narrowing of coordinates
zi_[zi_lower] = zi_[zi_lower] * t_reinf_layer_[zi_lower] / delta_
zi_[zi_upper] = 1 - (1 - zi_[zi_upper]) * t_reinf_layer_[zi_upper] / delta_
zi_[zi_inter] = t_reinf_layer_[zi_inter] + \
(zi_[zi_inter] - delta_) / (1 - 2 * delta_)\
* (1 - 2 * t_reinf_layer_[zi_inter])
print '--- constant edge elements transformation done ---'
# thickness is multiplied by the supplied zi coordinate
#
z_ = (zi_lower_ * self.scalefactor_delta_h + (zi_upper_ - zi_lower_) * zi_)
# coordinates of origin
#
X_0, Y_0, Z_0 = self.X0
print '--- geometric transformation done ---'
# multiply the local grid points with the real dimensions in order to obtain the
# global coordinates of the mushroof_part:
#
return c_[ X_0 + (xi_ * length_xy_tot) * self.scalefactor_length_xy,
Y_0 + (yi_ * length_xy_tot) * self.scalefactor_length_xy,
Z_0 + z_ * self.length_z ]
def _format_factor_matrix(self, dates, assets, filters, factors):
"""
Convert raw computed filters/factors into a DataFrame for public APIs.
Parameters
----------
dates : np.array[datetime64]
Row index for arrays in `filters` and `factors.`
assets : np.array[int64]
Column index for arrays in `filters` and `factors.`
filters : dict
Dict mapping filter names -> computed filters.
factors : dict
Dict mapping factor names -> computed factors.
Returns
-------
factor_matrix : pd.DataFrame
The indices of `factor_matrix` are as follows:
index : two-tiered MultiIndex of (date, asset).
For each date, we return a row for each asset that passed all
filters on that date.
columns : keys from `factor_data`
Each date/asset/factor triple contains the computed value of the given
factor on the given date for the given asset.
"""
# FUTURE OPTIMIZATION: Cythonize all of this.
# Boolean mask of values that passed all filters.
unioned = reduce(and_, itervalues(filters))
# Parallel arrays of (x,y) coords for (date, asset) pairs that passed
# all filters. Each entry here will correspond to a row in our output
# frame.
nonzero_xs, nonzero_ys = unioned.nonzero()
# Raw arrays storing (date, asset) pairs.
# These will form the index of our output frame.
raw_dates_index = empty_like(nonzero_xs, dtype='datetime64[ns]')
raw_assets_index = empty_like(nonzero_xs, dtype=int)
# Mapping from column_name -> array.
# This will be the `data` arg to our output frame.
columns = {
name: empty_like(nonzero_xs, dtype=factor.dtype)
for name, factor in iteritems(factors)
}
# We're going to iterate over `iteritems(columns)` a whole bunch of
# times down below. It's faster to construct iterate over a tuple of
# pairs.
columns_iter = tuple(iteritems(columns))
# This is tricky.
# unioned.sum(axis=1) gives us an array of the same size as `dates`
# containing, for each date, the number of assets that passed our
# filters on that date.
# Running this through add.accumulate gives us an array containing, for
# each date, the running total of the number of assets that passed our
# filters on or before that date.
# This means that (bounds[i - 1], bounds[i]) gives us the indices of
# the first and last rows in our output frame for each date in `dates`.
bounds = add.accumulate(unioned.sum(axis=1))
day_start = 0
for day_idx, day_end in enumerate(bounds):
day_bounds = slice(day_start, day_end)
column_indices = nonzero_ys[day_bounds]
raw_dates_index[day_bounds] = dates[day_idx]
raw_assets_index[day_bounds] = assets[column_indices]
for name, colarray in columns_iter:
colarray[day_bounds] = factors[name][day_idx, column_indices]
# Upper bound of current row becomes lower bound for next row.
day_start = day_end
return DataFrame(
data=columns,
index=MultiIndex.from_arrays(
[
raw_dates_index,
# FUTURE OPTIMIZATION:
# Avoid duplicate lookups by grouping and only looking up
# each unique sid once.
self._finder.retrieve_all(raw_assets_index),
],
)
).tz_localize('UTC', level=0)
def _format_factor_matrix(self, dates, assets, filters, factors):
"""
Convert raw computed filters/factors into a DataFrame for public APIs.
Parameters
----------
dates : np.array[datetime64]
Row index for arrays in `filters` and `factors.`
assets : np.array[int64]
Column index for arrays in `filters` and `factors.`
filters : dict
Dict mapping filter names -> computed filters.
factors : dict
Dict mapping factor names -> computed factors.
Returns
-------
factor_matrix : pd.DataFrame
The indices of `factor_matrix` are as follows:
index : two-tiered MultiIndex of (date, asset).
For each date, we return a row for each asset that passed all
filters on that date.
columns : keys from `factor_data`
Each date/asset/factor triple contains the computed value of the given
factor on the given date for the given asset.
"""
# FUTURE OPTIMIZATION: Cythonize all of this.
# Boolean mask of values that passed all filters.
unioned = reduce(and_, itervalues(filters))
# Parallel arrays of (x,y) coords for (date, asset) pairs that passed
# all filters. Each entry here will correspond to a row in our output
# frame.
nonzero_xs, nonzero_ys = unioned.nonzero()
# Raw arrays storing (date, asset) pairs.
# These will form the index of our output frame.
raw_dates_index = empty_like(nonzero_xs, dtype='datetime64[ns]')
raw_assets_index = empty_like(nonzero_xs, dtype=int)
# Mapping from column_name -> array.
# This will be the `data` arg to our output frame.
columns = {
name: empty_like(nonzero_xs, dtype=factor.dtype)
for name, factor in iteritems(factors)
}
# We're going to iterate over `iteritems(columns)` a whole bunch of
# times down below. It's faster to construct iterate over a tuple of
# pairs.
columns_iter = tuple(iteritems(columns))
# This is tricky.
# unioned.sum(axis=1) gives us an array of the same size as `dates`
# containing, for each date, the number of assets that passed our
# filters on that date.
# Running this through add.accumulate gives us an array containing, for
# each date, the running total of the number of assets that passed our
# filters on or before that date.
# This means that (bounds[i - 1], bounds[i]) gives us the indices of
# the first and last rows in our output frame for each date in `dates`.
bounds = add.accumulate(unioned.sum(axis=1))
day_start = 0
for day_idx, day_end in enumerate(bounds):
day_bounds = slice(day_start, day_end)
column_indices = nonzero_ys[day_bounds]
raw_dates_index[day_bounds] = dates[day_idx]
raw_assets_index[day_bounds] = assets[column_indices]
for name, colarray in columns_iter:
colarray[day_bounds] = factors[name][day_idx, column_indices]
# Upper bound of current row becomes lower bound for next row.
day_start = day_end
return DataFrame(
data=columns,
index=MultiIndex.from_arrays(
[
raw_dates_index,
# FUTURE OPTIMIZATION:
# Avoid duplicate lookups by grouping and only looking up
# each unique sid once.
self._finder.retrieve_all(raw_assets_index),
], )).tz_localize('UTC', level=0)
def cumtrapz(y, x=None, dx=1.0, axis=-1, initial=None):
"""
Cumulatively integrate y(x) using the composite trapezoidal rule.
Parameters
----------
y : array_like
Values to integrate.
x : array_like, optional
The coordinate to integrate along. If None (default), use spacing `dx`
between consecutive elements in `y`.
dx : int, optional
Spacing between elements of `y`. Only used if `x` is None.
axis : int, optional
Specifies the axis to cumulate. Default is -1 (last axis).
initial : scalar, optional
If given, uses this value as the first value in the returned result.
Typically this value should be 0. Default is None, which means no
value at ``x[0]`` is returned and `res` has one element less than `y`
along the axis of integration.
Returns
-------
res : ndarray
The result of cumulative integration of `y` along `axis`.
If `initial` is None, the shape is such that the axis of integration
has one less value than `y`. If `initial` is given, the shape is equal
to that of `y`.
See Also
--------
numpy.cumsum, numpy.cumprod
quad: adaptive quadrature using QUADPACK
romberg: adaptive Romberg quadrature
quadrature: adaptive Gaussian quadrature
fixed_quad: fixed-order Gaussian quadrature
dblquad: double integrals
tplquad: triple integrals
romb: integrators for sampled data
ode: ODE integrators
odeint: ODE integrators
Examples
--------
>>> from scipy import integrate
>>> import matplotlib.pyplot as plt
>>> x = np.linspace(-2, 2, num=20)
>>> y = x
>>> y_int = integrate.cumtrapz(y, x, initial=0)
>>> plt.plot(x, y_int, 'ro', x, y[0] + 0.5 * x**2, 'b-')
>>> plt.show()
"""
y = asarray(y)
if x is None:
d = dx
else:
d = diff(x, axis=axis)
nd = len(y.shape)
slice1 = tupleset((slice(None),)*nd, axis, slice(1, None))
slice2 = tupleset((slice(None),)*nd, axis, slice(None, -1))
res = add.accumulate(d * (y[slice1] + y[slice2]) / 2.0, axis)
if initial is not None:
if not np.isscalar(initial):
raise ValueError("`initial` parameter should be a scalar.")
shape = list(res.shape)
shape[axis] = 1
res = np.concatenate([np.ones(shape, dtype=res.dtype) * initial, res],
axis=axis)
return res
def __call__(self, points):
'''Return the global coordinates of the supplied local points.
'''
# number of local grid points for each coordinate direction
# NOTE: values must range between 0 and 1
#
xi_, yi_, zi_ = points[:, 0], points[:, 1], points[:, 2]
# insert imperfection (shift the middle node of the shell upwards)
imp = self.z_imperfection_factor
zi_ += imp * xi_ + imp * yi_ - 2 * imp * xi_ * yi_
# size of total structure
#
# @todo: move to class definition of "mushroof_model" and send to "__call__"
scale_size = self.scale_size
# @todo: add "_quarter" (see above)
length_xy_tot = self.length_xy_quarter * scale_size
n_elems_xy_quarter = self.n_elems_xy_quarter
# print 'HPShell n_elems_xy_quarter', n_elems_xy_quarter
# distance from origin for each mushroof_part
#
def d_origin_fn(self, coords):
if self.mushroof_part == 'quarter':
return coords
if self.mushroof_part == 'one':
return abs(2.0 * coords - 1.0)
# @todo: corresponding "scale_factor" needs to be added
# in order for this to work
if self.mushroof_part == 'four':
return where(coords < 0.5, abs(4 * coords - 1),
abs(-4 * coords + 3))
# element at column shift
#
if self.shift_elems == True:
# define the origin for each model part
#
def origin_fn(self, coords):
if self.mushroof_part == 'quarter':
return zeros_like(coords)
if self.mushroof_part == 'one':
return ones_like(xi_) * 0.5
if self.mushroof_part == 'four':
return where(coords < 0.5, 0.25, 0.75)
def piecewise_linear_fn(x, x_fix_arr_, y_fix_arr_):
'''creates a piecewise linear_fn going through the fix_points
values need to be normed running between 0..1
and values have to be unique'''
x_fix_arr_ = hstack((0, x_fix_arr_, 1))
y_fix_arr_ = hstack((0, y_fix_arr_, 1))
rbf_fn_ = Rbf(x_fix_arr_, y_fix_arr_,
function='linear') #rbf has to be linear
return rbf_fn_(x)
# define origin for quarter
#
xi_origin_arr_ = origin_fn(self, xi_)
yi_origin_arr_ = origin_fn(self, yi_)
# print 'xi_origin_arr_', xi_origin_arr_
# delta towards origin
#
xi_delta_arr_ = (xi_ - xi_origin_arr_) * scale_size
yi_delta_arr_ = (yi_ - yi_origin_arr_) * scale_size
# print 'xi_delta_arr_', xi_delta_arr
# define sign
#
xi_sign_arr = where(xi_delta_arr_ == 0., 0.,
xi_delta_arr_ / abs(xi_delta_arr_))
yi_sign_arr = where(yi_delta_arr_ == 0., 0.,
yi_delta_arr_ / abs(yi_delta_arr_))
# print 'xi_sign_arr', xi_sign_arr
# fix points defined in shift array as normelized values
#
x_fix_ = self.shift_array[:, 0] / self.length_xy_quarter
# print 'x_fix_', x_fix_
y_fix_ = self.shift_array[:, 1] / self.length_xy_quarter
n_fix_ = add.accumulate(self.shift_array[:,
2]) / n_elems_xy_quarter
# print 'add.accumulate( self.shift_array[:, 2] )', add.accumulate( self.shift_array[:, 2] )
# print 'n_fix_', n_fix_
# print 'piecewise_linear_fn', piecewise_linear_fn( abs( xi_delta_arr_ ),
# n_fix_,
# x_fix_ ) / scale_size
# new xi_
#
xi_ = xi_origin_arr_ + xi_sign_arr * piecewise_linear_fn(
abs(xi_delta_arr_), n_fix_, x_fix_) / scale_size
# print 'xi_new', xi_
# new yi
#
yi_ = yi_origin_arr_ + yi_sign_arr * piecewise_linear_fn(
abs(yi_delta_arr_), n_fix_, y_fix_) / scale_size
#-------------------------------------------------------------------------------------
# values are used to calculate the z-coordinate using RBF-function of the quarter
# (= values of the distance to the origin as absolute value)
#
xi_rbf_ = d_origin_fn(self, xi_)
# print 'xi_rbf_', xi_rbf_
yi_rbf_ = d_origin_fn(self, yi_)
# get the z-value at the supplied local grid points
# of the lower face
#
zi_lower_ = self.rbf_l_(xi_rbf_, yi_rbf_)
# get the z-value at the supplied local grid points
# of the upper face
#
zi_upper_ = self.rbf_u_(xi_rbf_, yi_rbf_)
# constant edge element transformation
#
if self.const_reinf_layer_elem == True:
# arrange and check data
#
if self.t_reinf_layer > self.t_shell / 2. or self.n_elems_z < 3:
print '--- constant edge element transformation canceled ---'
print 'the following condition needs to be fullfilled: \n'
print 'self.t_reinf_layer <= self.t_shell/2 and self.n_elems_z >= 3'
else:
n_elems_z = float(self.n_elems_z)
# normed thickness will evaluate as t_reinf_layer at each element
t_reinf_layer_ = self.t_reinf_layer / self.length_z / (
zi_upper_ - zi_lower_)
# zi_old set off from top which needs to be shifted
delta_ = self.n_elems_reinf_layer / n_elems_z
# get upper, lower and internal coordinates, that need to be shifted
zi_lower = where(zi_ <= delta_)
zi_upper = where(abs(1 - zi_) <= delta_ + 1e-10)
zi_inter = where(abs(zi_ - 0.5) < 0.5 - (delta_ + 1e-10))
# narrowing of coordinates
zi_[zi_lower] = zi_[zi_lower] * t_reinf_layer_[
zi_lower] / delta_
zi_[zi_upper] = 1 - (
1 - zi_[zi_upper]) * t_reinf_layer_[zi_upper] / delta_
zi_[zi_inter] = t_reinf_layer_[zi_inter] + \
(zi_[zi_inter] - delta_) / (1 - 2 * delta_)\
* (1 - 2 * t_reinf_layer_[zi_inter])
print '--- constant edge elements transformation done ---'
# thickness is multiplied by the supplied zi coordinate
#
z_ = (zi_lower_ * self.scalefactor_delta_h +
(zi_upper_ - zi_lower_) * zi_)
# coordinates of origin
#
X_0, Y_0, Z_0 = self.X0
print '--- geometric transformation done ---'
# multiply the local grid points with the real dimensions in order to obtain the
# global coordinates of the mushroof_part:
#
return c_[X_0 + (xi_ * length_xy_tot) * self.scalefactor_length_xy,
Y_0 + (yi_ * length_xy_tot) * self.scalefactor_length_xy,
Z_0 + z_ * self.length_z]
def load(self):
"""
Attempts to load candidate data from the file, performs file consistency checks if the
debug flag is set to true. Much of this code has been extracted from PRESTO by Scott Ransom.
Please see:
http://www.cv.nrao.edu/~sransom/presto/
https://github.com/scottransom/presto
Parameters:
N/A
Return:
N/A
"""
infile = open(self.cand, "rb")
# The code below appears to have been taken from Presto. So it maybe
# helpful to look at the Presto github repository (see above) to get a better feel
# for what this code is doing. I certainly have no idea what is going on. Although
# data is being unpacked in a specific order.
swapchar = '<' # this is little-endian
data = infile.read(5*4)
testswap = struct.unpack(swapchar+"i"*5, data)
# This is a hack to try and test the endianness of the data.
# None of the 5 values should be a large positive number.
if (fabs(asarray(testswap))).max() > 100000:
swapchar = '>' # this is big-endian
(self.numdms, self.numperiods, self.numpdots, self.nsub, self.npart) = struct.unpack(swapchar+"i"*5, data)
(self.proflen, self.numchan, self.pstep, self.pdstep, self.dmstep, self.ndmfact, self.npfact) = struct.unpack(swapchar+"i"*7, infile.read(7*4))
self.filenm = infile.read(struct.unpack(swapchar+"i", infile.read(4))[0])
self.candnm = infile.read(struct.unpack(swapchar+"i", infile.read(4))[0])
self.telescope = infile.read(struct.unpack(swapchar+"i", infile.read(4))[0])
self.pgdev = infile.read(struct.unpack(swapchar+"i", infile.read(4))[0])
test = infile.read(16)
has_posn = 1
for ii in range(16):
if test[ii] not in '0123456789:.-\0':
has_posn = 0
break
if has_posn:
self.rastr = test[:test.find('\0')]
test = infile.read(16)
self.decstr = test[:test.find('\0')]
(self.dt, self.startT) = struct.unpack(swapchar+"dd", infile.read(2*8))
else:
self.rastr = "Unknown"
self.decstr = "Unknown"
(self.dt, self.startT) = struct.unpack(swapchar+"dd", test)
(self.endT, self.tepoch, self.bepoch, self.avgvoverc, self.lofreq,self.chan_wid, self.bestdm) = struct.unpack(swapchar+"d"*7, infile.read(7*8))
(self.topo_pow, tmp) = struct.unpack(swapchar+"f"*2, infile.read(2*4))
(self.topo_p1, self.topo_p2, self.topo_p3) = struct.unpack(swapchar+"d"*3,infile.read(3*8))
(self.bary_pow, tmp) = struct.unpack(swapchar+"f"*2, infile.read(2*4))
(self.bary_p1, self.bary_p2, self.bary_p3) = struct.unpack(swapchar+"d"*3,infile.read(3*8))
(self.fold_pow, tmp) = struct.unpack(swapchar+"f"*2, infile.read(2*4))
(self.fold_p1, self.fold_p2, self.fold_p3) = struct.unpack(swapchar+"d"*3,infile.read(3*8))
(self.orb_p, self.orb_e, self.orb_x, self.orb_w, self.orb_t, self.orb_pd,self.orb_wd) = struct.unpack(swapchar+"d"*7, infile.read(7*8))
self.dms = asarray(struct.unpack(swapchar+"d"*self.numdms,infile.read(self.numdms*8)))
if self.numdms==1:
self.dms = self.dms[0]
self.periods = asarray(struct.unpack(swapchar + "d" * self.numperiods,infile.read(self.numperiods*8)))
self.pdots = asarray(struct.unpack(swapchar + "d" * self.numpdots,infile.read(self.numpdots*8)))
self.numprofs = self.nsub * self.npart
if (swapchar=='<'): # little endian
self.profs = zeros((self.npart, self.nsub, self.proflen), dtype='d')
for ii in range(self.npart):
for jj in range(self.nsub):
try:
self.profs[ii,jj,:] = fromfile(infile, float64, self.proflen)
except Exception: # Catch *all* exceptions.
pass
#print ""
else:
self.profs = asarray(struct.unpack(swapchar+"d"*self.numprofs*self.proflen,infile.read(self.numprofs*self.proflen*8)))
self.profs = reshape(self.profs, (self.npart, self.nsub, self.proflen))
self.binspersec = self.fold_p1 * self.proflen
self.chanpersub = self.numchan / self.nsub
self.subdeltafreq = self.chan_wid * self.chanpersub
self.hifreq = self.lofreq + (self.numchan-1) * self.chan_wid
self.losubfreq = self.lofreq + self.subdeltafreq - self.chan_wid
self.subfreqs = arange(self.nsub, dtype='d')*self.subdeltafreq + self.losubfreq
self.subdelays_bins = zeros(self.nsub, dtype='d')
self.killed_subbands = []
self.killed_intervals = []
self.pts_per_fold = []
# Note: a foldstats struct is read in as a group of 7 doubles
# the correspond to, in order:
# numdata, data_avg, data_var, numprof, prof_avg, prof_var, redchi
self.stats = zeros((self.npart, self.nsub, 7), dtype='d')
for ii in range(self.npart):
currentstats = self.stats[ii]
for jj in range(self.nsub):
if (swapchar=='<'): # little endian
try:
currentstats[jj] = fromfile(infile, float64, 7)
except Exception: # Catch *all* exceptions.
pass
#print ""
else:
try:
currentstats[jj] = asarray(struct.unpack(swapchar+"d"*7,infile.read(7*8)))
except Exception: # Catch *all* exceptions.
pass
#print ""
self.pts_per_fold.append(self.stats[ii][0][0]) # numdata from foldstats
self.start_secs = add.accumulate([0]+self.pts_per_fold[:-1])*self.dt
self.pts_per_fold = asarray(self.pts_per_fold)
self.mid_secs = self.start_secs + 0.5*self.dt*self.pts_per_fold
if (not self.tepoch==0.0):
self.start_topo_MJDs = self.start_secs/86400.0 + self.tepoch
self.mid_topo_MJDs = self.mid_secs/86400.0 + self.tepoch
if (not self.bepoch==0.0):
self.start_bary_MJDs = self.start_secs/86400.0 + self.bepoch
self.mid_bary_MJDs = self.mid_secs/86400.0 + self.bepoch
self.Nfolded = add.reduce(self.pts_per_fold)
self.T = self.Nfolded*self.dt
self.avgprof = (self.profs/self.proflen).sum()
self.varprof = self.calc_varprof()
self.barysubfreqs = self.subfreqs
infile.close()
# If explicit debugging required.
if(self.debug):
# If candidate file is invalid in some way...
if(self.isValid()==False):
print "Invalid PFD candidate: ",self.cand
raise Exception("Invalid PFD candidate: PFDFile.py (Line 214).")
# Candidate file is valid.
else:
print "Candidate file valid."
self.profile = array(self.getprofile())
# Just go directly to feature generation without checks.
else:
self.out( "Candidate validity checks skipped.","")
self.profile = array(self.getprofile())
def _format_factor_matrix(self,
dates,
assets,
filter_data,
factor_data,
factor_names):
"""
Convert raw computed filters/factors into a DataFrame for public APIs.
Parameters
----------
dates : np.array[datetime64]
Index for raw data in filter_data/factor_data.
assets : np.array[int64]
Column labels for raw data in filter_data/factor_data.
filter_data : list[ndarray[bool]]
Raw filters data.
factor_data : list[ndarray]
Raw factor data.
factor_names : list[str]
Names of factors to use as keys.
Returns
-------
factor_matrix : pd.DataFrame
A DataFrame with the following indices:
index : two-tiered MultiIndex of (date, asset). For each date, we
return a row for each asset that passed all filters on that
date.
columns : keys from `factor_data`
Each date/asset/factor triple contains the computed value of the given
factor on the given date for the given asset.
"""
# FUTURE OPTIMIZATION: Cythonize all of this.
# Boolean mask of values that passed all filters.
unioned = reduce(and_, filter_data)
# Parallel arrays of (x,y) coords for all date/asset pairs that passed
# all filters. Each entry here will correspond to a row in our output
# frame.
nonzero_xs, nonzero_ys = unioned.nonzero()
raw_dates_index = empty_like(nonzero_xs, dtype='datetime64[ns]')
raw_assets_index = empty_like(nonzero_xs, dtype=int)
factor_outputs = [
empty_like(nonzero_xs, dtype=factor.dtype)
for factor in factor_data
]
# This is tricky.
# unioned.sum(axis=1) gives us an array of the same size as `dates`
# containing, for each date, the number of assets that passed our
# filters on that date.
# Running this through add.accumulate gives us an array containing, for
# each date, the running total of the number of assets that passed our
# filters on or before that date.
# This means that (bounds[i - 1], bounds[i]) gives us the slice bounds
# of rows in our output DataFrame corresponding to each date.
dt_start = 0
bounds = add.accumulate(unioned.sum(axis=1))
for dt_idx, dt_end in enumerate(bounds):
bounds = slice(dt_start, dt_end)
column_indices = nonzero_ys[bounds]
raw_dates_index[bounds] = dates[dt_idx]
raw_assets_index[bounds] = assets[column_indices]
for computed, output in zip(factor_data, factor_outputs):
output[bounds] = computed[dt_idx, column_indices]
# Upper bound of current row becomes lower bound for next row.
dt_start = dt_end
return DataFrame(
dict(zip(factor_names, factor_outputs)),
index=MultiIndex.from_arrays(
[raw_dates_index, raw_assets_index],
)
).tz_localize('UTC', level=0)
# ============ITEM-3========================================================
pca = PCA(n_components=0.9, svd_solver="full")
pca.fit(X_data_scaled)
print(
"Números de componentes do PCA para 90% de explicação da variância: ",
pca.n_components_)
plt.plot(pca.explained_variance_ratio_)
plt.title('Taxa de explicação da variância por componente')
plt.ylabel('Taxa de explicação da Variância')
plt.xlabel('Número de Componentes')
plt.show()
plt.plot(add.accumulate(pca.explained_variance_ratio_))
plt.title('Explicação da variância acumulada por componente')
plt.ylabel('Taxa de explicação acumulada da Variância')
plt.xlabel('Número de Componentes')
plt.show()
# Obtido graficamente
pca = PCA(n_components=15)
pca.fit(X_data_scaled)
X_data_pca = pca.transform(X_data_scaled)
# ============ITEM-4========================================================
shuffle_splitter = ShuffleSplit(n_splits=5,
test_size=0.3,
random_state=1234)
regressor = LinearRegression()
def _format_factor_matrix(self, dates, assets, filter_data, factor_data,
factor_names):
"""
Convert raw computed filters/factors into a DataFrame for public APIs.
Parameters
----------
dates : np.array[datetime64]
Index for raw data in filter_data/factor_data.
assets : np.array[int64]
Column labels for raw data in filter_data/factor_data.
filter_data : list[ndarray[bool]]
Raw filters data.
factor_data : list[ndarray]
Raw factor data.
factor_names : list[str]
Names of factors to use as keys.
Returns
-------
factor_matrix : pd.DataFrame
A DataFrame with the following indices:
index : two-tiered MultiIndex of (date, asset). For each date, we
return a row for each asset that passed all filters on that
date.
columns : keys from `factor_data`
Each date/asset/factor triple contains the computed value of the given
factor on the given date for the given asset.
"""
# FUTURE OPTIMIZATION: Cythonize all of this.
# Boolean mask of values that passed all filters.
unioned = reduce(and_, filter_data)
# Parallel arrays of (x,y) coords for all date/asset pairs that passed
# all filters. Each entry here will correspond to a row in our output
# frame.
nonzero_xs, nonzero_ys = unioned.nonzero()
raw_dates_index = empty_like(nonzero_xs, dtype='datetime64[ns]')
raw_assets_index = empty_like(nonzero_xs, dtype=int)
factor_outputs = [
empty_like(nonzero_xs, dtype=factor.dtype)
for factor in factor_data
]
# This is tricky.
# unioned.sum(axis=1) gives us an array of the same size as `dates`
# containing, for each date, the number of assets that passed our
# filters on that date.
# Running this through add.accumulate gives us an array containing, for
# each date, the running total of the number of assets that passed our
# filters on or before that date.
# This means that (bounds[i - 1], bounds[i]) gives us the slice bounds
# of rows in our output DataFrame corresponding to each date.
dt_start = 0
bounds = add.accumulate(unioned.sum(axis=1))
for dt_idx, dt_end in enumerate(bounds):
bounds = slice(dt_start, dt_end)
column_indices = nonzero_ys[bounds]
raw_dates_index[bounds] = dates[dt_idx]
raw_assets_index[bounds] = assets[column_indices]
for computed, output in zip(factor_data, factor_outputs):
output[bounds] = computed[dt_idx, column_indices]
# Upper bound of current row becomes lower bound for next row.
dt_start = dt_end
return DataFrame(
dict(zip(factor_names, factor_outputs)),
index=MultiIndex.from_arrays(
[raw_dates_index, raw_assets_index], )).tz_localize('UTC',
level=0)