def bayesian_blocks(tt, ttstart, ttstop, p0, bkg_integral_distribution=None):
"""
Divide a series of events characterized by their arrival time in blocks
of perceptibly constant count rate. If the background integral distribution
is given, divide the series in blocks where the difference with respect to
the background is perceptibly constant.
:param tt: arrival times of the events
:param ttstart: the start of the interval
:param ttstop: the stop of the interval
:param p0: the false positive probability. This is used to decide the penalization on the likelihood, so this
parameter affects the number of blocks
:param bkg_integral_distribution: (default: None) If given, the algorithm account for the presence of the background and
finds changes in rate with respect to the background
:return: the np.array containing the edges of the blocks
"""
# Verify that the input array is one-dimensional
tt = np.asarray(tt, dtype=float)
assert tt.ndim == 1
if bkg_integral_distribution is not None:
# Transforming the inhomogeneous Poisson process into an homogeneous one with rate 1,
# by changing the time axis according to the background rate
logger.debug("Transforming the inhomogeneous Poisson process to a homogeneous one with rate 1...")
t = np.array(bkg_integral_distribution(tt))
logger.debug("done")
# Now compute the start and stop time in the new system
tstart = bkg_integral_distribution(ttstart)
tstop = bkg_integral_distribution(ttstop)
else:
t = tt
tstart = ttstart
tstop = ttstop
# Create initial cell edges (Voronoi tessellation)
edges = np.concatenate([[t[0]],
0.5 * (t[1:] + t[:-1]),
[t[-1]]])
# Create the edges also in the original time system
edges_ = np.concatenate([[tt[0]],
0.5 * (tt[1:] + tt[:-1]),
[tt[-1]]])
# Create a lookup table to be able to transform back from the transformed system
# to the original one
lookup_table = {key: value for (key, value) in zip(edges, edges_)}
# The last block length is 0 by definition
block_length = tstop - edges
if np.sum((block_length <= 0)) > 1:
raise RuntimeError("Events appears to be out of order! Check for order, or duplicated events.")
N = t.shape[0]
# arrays to store the best configuration
best = np.zeros(N, dtype=float)
last = np.zeros(N, dtype=int)
# eq. 21 from Scargle 2012
prior = 4 - np.log(73.53 * p0 * (N**-0.478))
logger.debug("Finding blocks...")
# This is where the computation happens. Following Scargle et al. 2012.
# This loop has been optimized for speed:
# * the expression for the fitness function has been rewritten to
# avoid multiple log computations, and to avoid power computations
# * the use of scipy.weave and numexpr has been evaluated. The latter
# gives a big gain (~40%) if used for the fitness function. No other
# gain is obtained by using it anywhere else
# Set numexpr precision to low (more than enough for us), which is
# faster than high
oldaccuracy = numexpr.set_vml_accuracy_mode('low')
numexpr.set_num_threads(1)
numexpr.set_vml_num_threads(1)
# Speed tricks: resolve once for all the functions which will be used
# in the loop
numexpr_evaluate = numexpr.evaluate
numexpr_re_evaluate = numexpr.re_evaluate
# Pre-compute this
aranges = np.arange(N+1, 0, -1)
for R in range(N):
br = block_length[R + 1]
T_k = block_length[:R + 1] - br # this looks like it is not used, but it actually is,
# inside the numexpr expression
# N_k: number of elements in each block
# This expression has been simplified for the case of
# unbinned events (i.e., one element in each block)
# It was:
#N_k = cumsum(x[:R + 1][::-1])[::-1]
# Now it is:
N_k = aranges[N - R:]
# where aranges has been pre-computed
# Evaluate fitness function
# This is the slowest part, which I'm speeding up by using
# numexpr. It provides a ~40% gain in execution speed.
# The first time we need to "compile" the expression in numexpr,
# all the other times we can reuse it
if R == 0:
fit_vec = numexpr_evaluate('''N_k * log(N_k/ T_k) ''',
optimization='aggressive', local_dict={'N_k': N_k, 'T_k': T_k})
else:
fit_vec = numexpr_re_evaluate(local_dict={'N_k': N_k, 'T_k': T_k})
A_R = fit_vec - prior # type: np.ndarray
A_R[1:] += best[:R]
i_max = A_R.argmax()
last[R] = i_max
best[R] = A_R[i_max]
numexpr.set_vml_accuracy_mode(oldaccuracy)
logger.debug("Done\n")
# Now peel off and find the blocks (see the algorithm in Scargle et al.)
change_points = np.zeros(N, dtype=int)
i_cp = N
ind = N
while True:
i_cp -= 1
change_points[i_cp] = ind
if ind == 0:
break
ind = last[ind - 1]
change_points = change_points[i_cp:]
edg = edges[change_points]
# Transform the found edges back into the original time system
if (bkg_integral_distribution is not None):
final_edges = map(lambda x: lookup_table[x], edg)
else:
final_edges = edg
# Now fix the first and last edge so that they are tstart and tstop
final_edges[0] = ttstart
final_edges[-1] = ttstop
return np.asarray(final_edges)
def bayesian_blocks_not_unique(tt, ttstart, ttstop, p0):
# Verify that the input array is one-dimensional
tt = np.asarray(tt, dtype=float)
assert tt.ndim == 1
# Now create the array of unique times
unique_t = np.unique(tt)
t = tt
tstart = ttstart
tstop = ttstop
# Create initial cell edges (Voronoi tessellation) using the unique time stamps
edges = np.concatenate([[tstart],
0.5 * (unique_t[1:] + unique_t[:-1]),
[tstop]])
# The last block length is 0 by definition
block_length = tstop - edges
if np.sum((block_length <= 0)) > 1:
raise RuntimeError("Events appears to be out of order! Check for order, or duplicated events.")
N = unique_t.shape[0]
# arrays to store the best configuration
best = np.zeros(N, dtype=float)
last = np.zeros(N, dtype=int)
# Pre-computed priors (for speed)
# eq. 21 from Scargle 2012
priors = 4 - np.log(73.53 * p0 * np.power(np.arange(1, N + 1), -0.478))
# Count how many events are in each Voronoi cell
x, _ = np.histogram(t, edges)
# Speed tricks: resolve once for all the functions which will be used
# in the loop
cumsum = np.cumsum
log = np.log
argmax = np.argmax
numexpr_evaluate = numexpr.evaluate
arange = np.arange
# Decide the step for reporting progress
incr = max(int(float(N) / 100.0 * 10), 1)
logger.debug("Finding blocks...")
# This is where the computation happens. Following Scargle et al. 2012.
# This loop has been optimized for speed:
# * the expression for the fitness function has been rewritten to
# avoid multiple log computations, and to avoid power computations
# * the use of scipy.weave and numexpr has been evaluated. The latter
# gives a big gain (~40%) if used for the fitness function. No other
# gain is obtained by using it anywhere else
# Set numexpr precision to low (more than enough for us), which is
# faster than high
oldaccuracy = numexpr.set_vml_accuracy_mode('low')
numexpr.set_num_threads(1)
numexpr.set_vml_num_threads(1)
for R in range(N):
br = block_length[R + 1]
T_k = block_length[:R + 1] - br
# N_k: number of elements in each block
# This expression has been simplified for the case of
# unbinned events (i.e., one element in each block)
# It was:
N_k = cumsum(x[:R + 1][::-1])[::-1]
# Now it is:
#N_k = arange(R + 1, 0, -1)
# Evaluate fitness function
# This is the slowest part, which I'm speeding up by using
# numexpr. It provides a ~40% gain in execution speed.
fit_vec = numexpr_evaluate('''N_k * log(N_k/ T_k) ''',
optimization='aggressive',
local_dict={'N_k': N_k, 'T_k': T_k})
p = priors[R]
A_R = fit_vec - p
A_R[1:] += best[:R]
i_max = argmax(A_R)
last[R] = i_max
best[R] = A_R[i_max]
pass
numexpr.set_vml_accuracy_mode(oldaccuracy)
logger.debug("Done\n")
# Now find blocks
change_points = np.zeros(N, dtype=int)
i_cp = N
ind = N
while True:
i_cp -= 1
change_points[i_cp] = ind
if ind == 0:
break
ind = last[ind - 1]
change_points = change_points[i_cp:]
finalEdges = edges[change_points]
return np.asarray(finalEdges)
compare_times(expression, 1)
sys.exit(0)
nexpr = 0
for expr in expressions:
nexpr += 1
compare_times(expr, nexpr)
print
if __name__ == "__main__":
import numexpr
numexpr.print_versions()
numexpr.set_vml_accuracy_mode("low")
numexpr.set_vml_num_threads(2)
if len(sys.argv) > 1:
expression = sys.argv[1]
print "expression-->", expression
compare(expression)
else:
compare()
tratios = numpy.array(numpy_ttime) / numpy.array(numexpr_ttime)
stratios = numpy.array(numpy_sttime) / numpy.array(numexpr_sttime)
ntratios = numpy.array(numpy_nttime) / numpy.array(numexpr_nttime)
print "*************** Numexpr vs NumPy speed-ups *******************"
# print "numpy total:", sum(numpy_ttime)/iterations
# print "numpy strided total:", sum(numpy_sttime)/iterations
"""arithmetic functions"""
import multiprocessing
import multiprocessing.pool
import warnings
import numpy as np
try:
import numexpr
numexpr.set_num_threads(multiprocessing.cpu_count())
numexpr.set_vml_num_threads(multiprocessing.cpu_count())
except ImportError:
warnings.warn('numexpr not detected, use `sudo pip install numexpr`')
numexpr = None
from . import arrays
#
# # main functions
#
def sum(X, axis):
"""compute sum of array along an axis
Parameters
----------
- X: array to compute sum of
- axis: int axis along which to compute sum
"""
def bayesian_blocks(tt, ttstart, ttstop, p0, bkgIntegralDistr=None, myLikelihood=None):
"""Divide a series of events characterized by their arrival time in blocks
of perceptibly constant count rate. If the background integral distribution
is given, divide the series in blocks where the difference with respect to
the background is perceptibly constant.
Args:
tt (iterable): An iterable (list, numpy.array...) containing the arrival
time of the events.
NOTE: the input array MUST be time-ordered, and without
duplicated entries. To ensure this, you may execute the
following code:
tt_array = numpy.asarray(tt)
tt_array = numpy.unique(tt_array)
tt_array.sort()
before running the algorithm.
p0 (float): The probability of finding a variations (i.e., creating a new
block) when there is none. In other words, the probability of
a Type I error, i.e., rejecting the null-hypothesis when is
true. All found variations will have a post-trial significance
larger than p0.
bkgIntegralDistr (function, optional): the integral distribution for the
background counts. It must be a function of the form f(x),
which must return the integral number of counts expected from
the background component between time 0 and x.
Returns:
numpy.array: the edges of the blocks found
"""
# Verify that the input array is one-dimensional
tt = np.asarray(tt, dtype=float)
assert tt.ndim == 1
if (bkgIntegralDistr is not None):
# Transforming the inhomogeneous Poisson process into an homogeneous one with rate 1,
# by changing the time axis according to the background rate
logger.debug("Transforming the inhomogeneous Poisson process to a homogeneous one with rate 1...")
t = np.array(bkgIntegralDistr(tt))
logger.debug("done")
# Now compute the start and stop time in the new system
tstart = bkgIntegralDistr(ttstart)
tstop = bkgIntegralDistr(ttstop)
else:
t = tt
tstart = ttstart
tstop = ttstop
pass
# Create initial cell edges (Voronoi tessellation)
edges = np.concatenate([[tstart],
0.5 * (t[1:] + t[:-1]),
[tstop]])
# Create the edges also in the original time system
edges_ = np.concatenate([[ttstart],
0.5 * (tt[1:] + tt[:-1]),
[ttstop]])
# Create a lookup table to be able to transform back from the transformed system
# to the original one
lookupTable = {key: value for (key, value) in zip(edges, edges_)}
# The last block length is 0 by definition
block_length = tstop - edges
if np.sum((block_length <= 0)) > 1:
raise RuntimeError("Events appears to be out of order! Check for order, or duplicated events.")
N = t.shape[0]
# arrays to store the best configuration
best = np.zeros(N, dtype=float)
last = np.zeros(N, dtype=int)
best_new = np.zeros(N, dtype=float)
last_new = np.zeros(N, dtype=int)
# Pre-computed priors (for speed)
if (myLikelihood):
priors = myLikelihood.getPriors(N, p0)
else:
# eq. 21 from Scargle 2012
#priors = 4 - np.log(73.53 * p0 * np.power(np.arange(1, N + 1), -0.478))
priors = [4 - np.log(73.53 * p0 * N**(-0.478))] * N
pass
x = np.ones(N)
# Speed tricks: resolve once for all the functions which will be used
# in the loop
cumsum = np.cumsum
log = np.log
argmax = np.argmax
numexpr_evaluate = numexpr.evaluate
arange = np.arange
# Decide the step for reporting progress
incr = max(int(float(N) / 100.0 * 10), 1)
logger.debug("Finding blocks...")
# This is where the computation happens. Following Scargle et al. 2012.
# This loop has been optimized for speed:
# * the expression for the fitness function has been rewritten to
# avoid multiple log computations, and to avoid power computations
# * the use of scipy.weave and numexpr has been evaluated. The latter
# gives a big gain (~40%) if used for the fitness function. No other
# gain is obtained by using it anywhere else
times = []
TSs = []
# Set numexpr precision to low (more than enough for us), which is
# faster than high
oldaccuracy = numexpr.set_vml_accuracy_mode('low')
numexpr.set_num_threads(1)
numexpr.set_vml_num_threads(1)
for R in range(N):
br = block_length[R + 1]
T_k = block_length[:R + 1] - br
# N_k: number of elements in each block
# This expression has been simplified for the case of
# unbinned events (i.e., one element in each block)
# It was:
# N_k = cumsum(x[:R + 1][::-1])[::-1]
# Now it is:
N_k = arange(R + 1, 0, -1)
# Evaluate fitness function
# This is the slowest part, which I'm speeding up by using
# numexpr. It provides a ~40% gain in execution speed.
fit_vec = numexpr_evaluate('''N_k * log(N_k/ T_k) ''',
optimization='aggressive')
p = priors[R]
A_R = fit_vec - p
A_R[1:] += best[:R]
i_max = argmax(A_R)
last[R] = i_max
best[R] = A_R[i_max]
# if(myLikelihood):
# logger.debug("Maximum old: %i, Maximum new: %i" %(i_max,i_max_new))
# logger.debug("Best old: %s, Best new: %s" %(best[R],best_new[R]))
pass
numexpr.set_vml_accuracy_mode(oldaccuracy)
# if(myLikelihood):
# from operator import itemgetter
# index, element = max(enumerate(TSs), key=itemgetter(1))
# t1,t2 = times[index]
# print("Maximum TS is %s in time interval %s-%s" %(element,t1,t2))
#
# best = best_new
# last = last_new
# map(oneLoop,range(N))
logger.debug("Done\n")
# Now find blocks
change_points = np.zeros(N, dtype=int)
i_cp = N
ind = N
while True:
i_cp -= 1
change_points[i_cp] = ind
if ind == 0:
break
ind = last[ind - 1]
change_points = change_points[i_cp:]
edg = edges[change_points]
# Transform the found edges back into the original time system
if (bkgIntegralDistr is not None):
finalEdges = map(lambda x: lookupTable[x], edg)
else:
finalEdges = edg
pass
return np.asarray(finalEdges)
if expression:
compare_times(expression, 1)
sys.exit(0)
nexpr = 0
for expr in expressions:
nexpr += 1
compare_times(expr, nexpr)
print
if __name__ == '__main__':
import numexpr
numexpr.print_versions()
numexpr.set_vml_accuracy_mode('low')
numexpr.set_vml_num_threads(2)
if len(sys.argv) > 1:
expression = sys.argv[1]
print "expression-->", expression
compare(expression)
else:
compare()
tratios = numpy.array(numpy_ttime) / numpy.array(numexpr_ttime)
stratios = numpy.array(numpy_sttime) / numpy.array(numexpr_sttime)
ntratios = numpy.array(numpy_nttime) / numpy.array(numexpr_nttime)
print "*************** Numexpr vs NumPy speed-ups *******************"
# print "numpy total:", sum(numpy_ttime)/iterations
# print "numpy strided total:", sum(numpy_sttime)/iterations
filters='filters.hd5',
kurucz04='kurucz2004.grid.fits',
tlusty09='tlusty.lowres.grid.fits',
hstcovar='hst_whitedwarf_frac_covar.fits',
basel22='stellib_BaSeL_v2.2.grid.fits',
munari='atlas9-munari.hires.grid.fits',
btsettl='bt-settl.lowres.grid.fits'
# elodie31 = 'Elodie_v3.1.grid.fits'
)
# Make sure the configuration is coherent for the python installation
try:
import numexpr
if not __USE_NUMEXPR__:
numexpr.set_num_threads(1)
numexpr.set_vml_num_threads(1)
else:
numexpr.set_num_threads(__NTHREADS__)
numexpr.set_vml_num_threads(__NTHREADS__)
except ImportError:
__USE_NUMEXPR__ = False
try:
import tables
tables.parameters.MAX_NUMEXPR_THREADS = __NTHREADS__
tables.parameters.MAX_BLOSC_THREADS = __NTHREADS__
tables.set_blosc_max_threads(__NTHREADS__)
except ImportError:
pass
'''
Created on May 5, 2016
A collection of functions to compute derived variables, primarily for WRF but also in general.
@author: Andre R. Erler, GPL v3
'''
# external imports
from warnings import warn
from numexpr import evaluate, set_num_threads, set_vml_num_threads
# numexpr parallelisation: don't parallelize at this point!
set_num_threads(1); set_vml_num_threads(1)
# internal imports
from geodata.base import Variable, VariableError
from utils.constants import sig # used in calculation for clack body radiation
## helper functions
# net radiation balance using black-body radiation from skin temperature
def radiation_black(A, SW, LW, e, Ts, TSmax=None):
''' net radiation [W/m^2] at the surface: downwelling long and short wave minus upwelling terrestrial radiation '''
if TSmax is None:
# using average skin temperature for terrestrial long wave emission
warn('Using average skin temperature; diurnal min/max skin temperature is preferable due to strong nonlinearity.')
return evaluate('( ( 1 - A ) * SW ) + ( LW * e ) - ( e * sig * Ts**4 )')
else:
# using min/max skin temperature to account for nonlinearity
return evaluate('( ( 1 - A ) * SW ) + ( LW * e ) - ( e * sig * ( Ts**4 + TSmax**4 ) / 2 )')
# net radiation balance using accumulated quantities
def bayesian_blocks(tt, ttstart, ttstop, p0, bkg_integral_distribution=None):
"""
Divide a series of events characterized by their arrival time in blocks
of perceptibly constant count rate. If the background integral distribution
is given, divide the series in blocks where the difference with respect to
the background is perceptibly constant.
:param tt: arrival times of the events
:param ttstart: the start of the interval
:param ttstop: the stop of the interval
:param p0: the false positive probability. This is used to decide the penalization on the likelihood, so this
parameter affects the number of blocks
:param bkg_integral_distribution: (default: None) If given, the algorithm account for the presence of the background and
finds changes in rate with respect to the background
:return: the np.array containing the edges of the blocks
"""
# Verify that the input array is one-dimensional
tt = np.asarray(tt, dtype=float)
assert tt.ndim == 1
if bkg_integral_distribution is not None:
# Transforming the inhomogeneous Poisson process into an homogeneous one with rate 1,
# by changing the time axis according to the background rate
logger.debug(
"Transforming the inhomogeneous Poisson process to a homogeneous one with rate 1..."
)
t = np.array(bkg_integral_distribution(tt))
logger.debug("done")
# Now compute the start and stop time in the new system
tstart = bkg_integral_distribution(ttstart)
tstop = bkg_integral_distribution(ttstop)
else:
t = tt
tstart = ttstart
tstop = ttstop
# Create initial cell edges (Voronoi tessellation)
edges = np.concatenate([[t[0]], 0.5 * (t[1:] + t[:-1]), [t[-1]]])
# Create the edges also in the original time system
edges_ = np.concatenate([[tt[0]], 0.5 * (tt[1:] + tt[:-1]), [tt[-1]]])
# Create a lookup table to be able to transform back from the transformed system
# to the original one
lookup_table = {key: value for (key, value) in zip(edges, edges_)}
# The last block length is 0 by definition
block_length = tstop - edges
if np.sum((block_length <= 0)) > 1:
raise RuntimeError(
"Events appears to be out of order! Check for order, or duplicated events."
)
N = t.shape[0]
# arrays to store the best configuration
best = np.zeros(N, dtype=float)
last = np.zeros(N, dtype=int)
# eq. 21 from Scargle 2012
prior = 4 - np.log(73.53 * p0 * (N**-0.478))
logger.debug("Finding blocks...")
# This is where the computation happens. Following Scargle et al. 2012.
# This loop has been optimized for speed:
# * the expression for the fitness function has been rewritten to
# avoid multiple log computations, and to avoid power computations
# * the use of scipy.weave and numexpr has been evaluated. The latter
# gives a big gain (~40%) if used for the fitness function. No other
# gain is obtained by using it anywhere else
# Set numexpr precision to low (more than enough for us), which is
# faster than high
oldaccuracy = numexpr.set_vml_accuracy_mode('low')
numexpr.set_num_threads(1)
numexpr.set_vml_num_threads(1)
# Speed tricks: resolve once for all the functions which will be used
# in the loop
numexpr_evaluate = numexpr.evaluate
numexpr_re_evaluate = numexpr.re_evaluate
# Pre-compute this
aranges = np.arange(N + 1, 0, -1)
for R in range(N):
br = block_length[R + 1]
T_k = block_length[:R +
1] - br # this looks like it is not used, but it actually is,
# inside the numexpr expression
# N_k: number of elements in each block
# This expression has been simplified for the case of
# unbinned events (i.e., one element in each block)
# It was:
#N_k = cumsum(x[:R + 1][::-1])[::-1]
# Now it is:
N_k = aranges[N - R:]
# where aranges has been pre-computed
# Evaluate fitness function
# This is the slowest part, which I'm speeding up by using
# numexpr. It provides a ~40% gain in execution speed.
# The first time we need to "compile" the expression in numexpr,
# all the other times we can reuse it
if R == 0:
fit_vec = numexpr_evaluate('''N_k * log(N_k/ T_k) ''',
optimization='aggressive',
local_dict={
'N_k': N_k,
'T_k': T_k
})
else:
fit_vec = numexpr_re_evaluate(local_dict={'N_k': N_k, 'T_k': T_k})
A_R = fit_vec - prior # type: np.ndarray
A_R[1:] += best[:R]
i_max = A_R.argmax()
last[R] = i_max
best[R] = A_R[i_max]
numexpr.set_vml_accuracy_mode(oldaccuracy)
logger.debug("Done\n")
# Now peel off and find the blocks (see the algorithm in Scargle et al.)
change_points = np.zeros(N, dtype=int)
i_cp = N
ind = N
while True:
i_cp -= 1
change_points[i_cp] = ind
if ind == 0:
break
ind = last[ind - 1]
change_points = change_points[i_cp:]
edg = edges[change_points]
# Transform the found edges back into the original time system
if (bkg_integral_distribution is not None):
final_edges = map(lambda x: lookup_table[x], edg)
else:
final_edges = edg
# Now fix the first and last edge so that they are tstart and tstop
final_edges[0] = ttstart
final_edges[-1] = ttstop
return np.asarray(final_edges)
'''
Created on May 5, 2016
A collection of functions to compute derived variables, primarily for WRF but also in general.
@author: Andre R. Erler, GPL v3
'''
# external imports
from warnings import warn
from numexpr import evaluate, set_num_threads, set_vml_num_threads
# numexpr parallelisation: don't parallelize at this point!
set_num_threads(1)
set_vml_num_threads(1)
# internal imports
from geodata.base import Variable, VariableError
from utils.constants import sig # used in calculation for clack body radiation
## helper functions
# net radiation balance using black-body radiation from skin temperature
def radiation_black(A, SW, LW, e, Ts, TSmax=None):
''' net radiation [W/m^2] at the surface: downwelling long and short wave minus upwelling terrestrial radiation '''
if TSmax is None:
# using average skin temperature for terrestrial long wave emission
warn(
'Using average skin temperature; diurnal min/max skin temperature is preferable due to strong nonlinearity.'
)
return evaluate(
'( ( 1 - A ) * SW ) + ( LW * e ) - ( e * sig * Ts**4 )')
def bayesian_blocks_not_unique(tt, ttstart, ttstop, p0):
# Verify that the input array is one-dimensional
tt = np.asarray(tt, dtype=float)
assert tt.ndim == 1
# Now create the array of unique times
unique_t = np.unique(tt)
t = tt
tstart = ttstart
tstop = ttstop
# Create initial cell edges (Voronoi tessellation) using the unique time stamps
edges = np.concatenate([[tstart],
0.5 * (unique_t[1:] + unique_t[:-1]),
[tstop]])
# The last block length is 0 by definition
block_length = tstop - edges
if np.sum((block_length <= 0)) > 1:
raise RuntimeError("Events appears to be out of order! Check for order, or duplicated events.")
N = unique_t.shape[0]
# arrays to store the best configuration
best = np.zeros(N, dtype=float)
last = np.zeros(N, dtype=int)
# Pre-computed priors (for speed)
# eq. 21 from Scargle 2012
priors = 4 - np.log(73.53 * p0 * np.power(np.arange(1, N + 1), -0.478))
# Count how many events are in each Voronoi cell
x, _ = np.histogram(t, edges)
# Speed tricks: resolve once for all the functions which will be used
# in the loop
cumsum = np.cumsum
log = np.log
argmax = np.argmax
numexpr_evaluate = numexpr.evaluate
arange = np.arange
# Decide the step for reporting progress
incr = max(int(float(N) / 100.0 * 10), 1)
logger.debug("Finding blocks...")
# This is where the computation happens. Following Scargle et al. 2012.
# This loop has been optimized for speed:
# * the expression for the fitness function has been rewritten to
# avoid multiple log computations, and to avoid power computations
# * the use of scipy.weave and numexpr has been evaluated. The latter
# gives a big gain (~40%) if used for the fitness function. No other
# gain is obtained by using it anywhere else
# Set numexpr precision to low (more than enough for us), which is
# faster than high
oldaccuracy = numexpr.set_vml_accuracy_mode('low')
numexpr.set_num_threads(1)
numexpr.set_vml_num_threads(1)
with progress_bar(N) as progress:
for R in range(N):
br = block_length[R + 1]
T_k = block_length[:R + 1] - br
# N_k: number of elements in each block
# This expression has been simplified for the case of
# unbinned events (i.e., one element in each block)
# It was:
N_k = cumsum(x[:R + 1][::-1])[::-1]
# Now it is:
# N_k = arange(R + 1, 0, -1)
# Evaluate fitness function
# This is the slowest part, which I'm speeding up by using
# numexpr. It provides a ~40% gain in execution speed.
fit_vec = numexpr_evaluate('''N_k * log(N_k/ T_k) ''',
optimization='aggressive',
local_dict={'N_k': N_k, 'T_k': T_k})
p = priors[R]
A_R = fit_vec - p
A_R[1:] += best[:R]
i_max = argmax(A_R)
last[R] = i_max
best[R] = A_R[i_max]
progress.increase()
numexpr.set_vml_accuracy_mode(oldaccuracy)
logger.debug("Done\n")
# Now find blocks
change_points = np.zeros(N, dtype=int)
i_cp = N
ind = N
while True:
i_cp -= 1
change_points[i_cp] = ind
if ind == 0:
break
ind = last[ind - 1]
change_points = change_points[i_cp:]
finalEdges = edges[change_points]
return np.asarray(finalEdges)
import yfinance as yf
import streamlit as st
import numexpr as ne
import pandas as pd
#HERE: reset number of vml-threads
ne.set_vml_num_threads(4)
#App header
st.write("""
# STOCK PRICES APP
Shown are the stock closing price and volume for Google!
""")
#defining a ticker symbol
tickerSymbol = 'GOOGL'
#getting data from ticker
tickerData = yf.Ticker(tickerSymbol)
#getting historical prices from the ticker
tickerDf = tickerData.history(period="1d",
start='2010-05-31',
end='2020-05-31')
st.line_chart(tickerDf.Close)
st.line_chart(tickerDf.Volume)
t0 = time()
z = 2 * y + 4 * x
t1 = time()
gbs = working_set_GB / (t1 - t0)
print("Time for an algebraic expression: %.3f s / %.3f GB/s" %
(t1 - t0, gbs))
t0 = time()
z = np.sin(x)**2 + np.cos(y)**2
t1 = time()
gbs = working_set_GB / (t1 - t0)
print("Time for a transcendental expression: %.3f s / %.3f GB/s" %
(t1 - t0, gbs))
if ne.use_vml:
ne.set_vml_num_threads(1)
ne.set_num_threads(8)
print("NumExpr version: %s, Using MKL ver. %s, Num threads: %s" %
(ne.__version__, ne.get_vml_version(), ne.nthreads))
else:
ne.set_num_threads(8)
print("NumExpr version: %s, Not Using MKL, Num threads: %s" %
(ne.__version__, ne.nthreads))
t0 = time()
ne.evaluate('2*y + 4*x', out=z)
t1 = time()
gbs = working_set_GB / (t1 - t0)
print("Time for an algebraic expression: %.3f s / %.3f GB/s" %
(t1 - t0, gbs))
def _evaluate_utilities(self,
expressions: Union[ExpressionGroup,
ExpressionSubGroup],
n_threads: int = None,
logger: Logger = None,
allow_casting=True) -> DataFrame:
if self._decision_units is None:
raise ModelNotReadyError(
"Decision units must be set before evaluating utility expressions"
)
if n_threads is None:
n_threads = cpu_count()
row_index = self._decision_units
col_index = self.choices
# if debug, get index location of corresponding id
debug_label = None
debug_expr = []
debug_results = []
if self.debug_id:
debug_label = row_index.get_loc(self.debug_id)
utilities = self._partial_utilities.values
# Prepare locals, including scalar, vector, and matrix variables that don't need any further processing.
shared_locals = {
NAN_STR: np.nan,
OUT_STR: utilities,
NEG_INF_STR: NEG_INF_VAL
}
for name in expressions.itersimple():
if name in shared_locals:
continue
symbol = self._scope[name]
shared_locals[name] = symbol._get()
ne.set_num_threads(n_threads)
ne.set_vml_num_threads(n_threads)
casting_rule = 'same_kind' if allow_casting else 'safe'
for expr in expressions:
if logger is not None:
logger.debug(f"Evaluating expression `{expr.raw}`")
# TODO: Add error handling
choice_mask = self._make_column_mask(expr.filter_)
local_dict = shared_locals.copy(
) # Make a shallow copy of the shared symbols
# Add in any dict literals, expanding them to cover all choices
expr._prepare_dict_literals(col_index, local_dict)
# Evaluate any chains on-the-fly
for symbol_name, usages in expr.chains.items():
symbol = self._scope[symbol_name]
for substitution, chain_info in usages.items():
data = symbol._get(chain_info=chain_info)
local_dict[substitution] = data
self._kernel_eval(expr.transformed,
local_dict,
utilities,
choice_mask,
casting_rule=casting_rule)
# save each expression and values for a specific od pair
if self.debug_id:
debug_expr.append(expr.raw)
debug_results.append(utilities[debug_label].copy())
nans = np.isnan(utilities)
n_nans = nans.sum()
if n_nans > 0:
raise UtilityBoundsError(
f"Found {n_nans} cells in utility table with NaN")
if self.debug_id: # expressions.tolist() doesn't work...
self.debug_results = DataFrame(debug_results,
index=debug_expr,
columns=col_index)
return DataFrame(utilities, index=row_index, columns=col_index)
#!/usr/bin/env python # Import python modules import sys, os, glob, copy, itertools from natsort import natsorted import numpy as np import pandas as pd import numexpr as ne # Global Variables DELIMITER = '__' MAX_PROCESSES = 7 PARALLEL = 0 ne.set_vml_num_threads(MAX_PROCESSES) from .graph_settings import set_settings, permute_settings #,get_settings, from .graph_functions import structure, terms, save #,analysis from .texify import Texify #,scinotation from .dictionary import _set, _get, _pop, _has, _update, _permute from .load_dump import setup, path_join #, load,dump,path_split # Logging import logging, logging.handlers log = 'info' rootlogger = logging.getLogger()
# -*- coding: utf-8 -*-
import numpy as np
import scipy.optimize
import numexpr as ne
log2pi = np.log(2 * np.pi)
ne.set_vml_num_threads(16)
class VariationalParams():
def __init__(self, zeta, phi, lambd, nu_sq, doc_counts):
self.zeta = zeta
self.phi = phi
self.lambd = lambd
self.nu_sq = nu_sq
self.weighted_sum_phi = doc_counts.dot(phi)
def update_ws_phi(self, doc_counts):
self.weighted_sum_phi = doc_counts.dot(self.phi) #Must be called every time phi is changed
def __str__(self):
return "zeta: " + str(self.zeta) + "; phi: " + str(self.phi) + \
"lambda: " + str(self.lambd) + "; nu_sq: " + str(self.nu_sq)
"""First derivative of f_lambda with respect to lambda.
Uses the first parameter (passed by the optimizer) as lambda."""
def f_dlambda(lambd, v_params, m_params, doc, counts, N):
mu=m_params.mu
nu_sq=v_params.nu_sq
zeta=v_params.zeta
