def normal_test_case():
'''
Runs a test case with simulated data from a normal distribution.
'''
obs, fa, dur = [], [], []
for n in range(15):
d, f, o = make_test_data(
5, split=min(plt.rand()*50+120, 170),
intercept=plt.rand()*50 + 225,
slope1=1 + plt.randn()/0.75, slope2=plt.randn()/.75)
obs.append(o+n)
fa.append(f)
dur.append(d)
plt.plot(f, d, 'o', alpha=0.1)
dur, fa, obs = (np.hstack(dur)[:, np.newaxis],
np.hstack(fa)[:, np.newaxis],
np.hstack(obs)[:, np.newaxis])
dur_mean = dur.mean()
dur_std = dur.std()
dur = (dur-dur_mean)/dur_std
m = normal_model(dur, fa, obs)
trace = sample_model(m, 5000)
predict(trace, 5, 2500, {'mean': dur_mean, 'std': dur_std})
plt.figure()
traceplot(trace, 2, 2500)
return dur, fa, obs, (dur_mean, dur_std), trace
def gamma_test_case():
'''
Runs a test case with simulated data from a normal distribution.
'''
obs, fa, dur = [], [], []
delta_angle = np.arange(180)
for n in range(15):
mode = piecewise_predictor(
delta_angle,
100 + plt.randn()*20,
250 + plt.randn()*20,
1 + plt.randn()/2.0,
-1 + plt.randn()/2.0)
a, b = np_gamma_params(mode, 10)
for _ in range(10):
d = gamma.rvs(a=a, scale=1.0/b)
fa.append(delta_angle)
dur.append(d)
obs.append(d*0+n)
dur, fa, obs = np.concatenate(dur), np.concatenate(fa), np.concatenate(obs)
m = gamma_model(dur, fa, obs.astype(int))
trace = sample_model(m, 5000)
predict(trace, 5, 2500 )
plt.figure()
traceplot(trace, 2, 2500)
return dur, fa, obs, trace
def __init__(self, numstates, numclasses):
self.numstates = numstates
self.numclasses = numclasses
self.numparams = self.numclasses * self.numstates\
+ self.numstates\
+ self.numstates**2
self.params = zeros(self.numparams, dtype=float)
self.logEmissionProbs = \
self.params[:self.numclasses * self.numstates].reshape(
self.numstates, self.numclasses)
self.logInitProbs = self.params[self.numclasses * self.numstates:
self.numclasses * self.numstates+
self.numstates]
self.logTransitionProbs = self.params[-self.numstates**2:].reshape(
self.numstates, self.numstates)
self.logEmissionProbs[:] = \
ones((self.numstates, self.numclasses),dtype=numpy.double)\
/numpy.double(self.numclasses) +\
randn(self.numstates, self.numclasses)*0.001
self.logEmissionProbs /= self.logEmissionProbs.sum(1)[:,newaxis]
self.logEmissionProbs = log(self.logEmissionProbs)
self.logInitProbs[:] = ones(self.numstates, dtype=float) \
/ self.numstates+\
randn(self.numstates)*0.001
self.logInitProbs /= self.logInitProbs.sum()
self.logInitProbs[:] = log(self.logInitProbs)
self.logTransitionProbs[:] = ones((self.numstates, self.numstates),
dtype=float)/self.numstates+\
randn(self.numstates,self.numstates)*0.001
self.logTransitionProbs /= self.logTransitionProbs.sum(1)[:,newaxis]
self.logTransitionProbs[:] = log(self.logTransitionProbs)
def draw_line(self, x_axis, y_axis):
X = randn(x_axis)
Y = randn(y_axis)
# it shows lines for random X and Y values
plt.scatter(X, Y, color='b')
plt.plot(X, Y)
plt.xlabel('X Axis')
plt.ylabel('Y Axis')
plt.title("Lines with scatter plot")
plt.show()
def makeScatter():
x1 = randn(200)
y1 = randn(200)
x2 = randn(200)
y2 = randn(200)
plt.scatter(x1, y1, color='r', s=100, label="dot1")
plt.scatter(x2, y2, color='c', s=10, label="dot2")
plt.xlabel("x")
plt.ylabel("y")
plt.legend()
plt.show()
def OnGenShiftmapQuad(self, event):
from PYME.Analysis.points import twoColour, twoColourPlot
from PYME.IO.MetaDataHandler import get_camera_roi_origin
pipeline = self.visFr.pipeline
vs = pipeline.mdh.voxelsize_nm
roi_x0, roi_y0 = get_camera_roi_origin(pipeline.mdh)
x0 = (roi_x0) * vs[0]
y0 = (roi_y0) * vs[1]
lx = len(pipeline.filter['x'])
bbox = None #[0,(pipeline.mdh['Camera.ROIWidth'] + 1)*vs[0], 0,(pipeline.mdh['Camera.ROIHeight'] + 1)*vs[1]]
dx, dy, spx, spy, good = twoColour.genShiftVectorFieldQ(
pipeline.filter['x'] + .1 * pylab.randn(lx) + x0,
pipeline.filter['y'] + .1 * pylab.randn(lx) + y0,
pipeline.filter['fitResults_dx'],
pipeline.filter['fitResults_dy'],
pipeline.filter['fitError_dx'],
pipeline.filter['fitError_dy'],
bbox=bbox)
#twoColourPlot.PlotShiftField(dx, dy, spx, spy)
twoColourPlot.PlotShiftField2(spx,
spy,
pipeline.mdh['Splitter.Channel0ROI'][2:],
voxelsize=vs)
twoColourPlot.PlotShiftResiduals(pipeline['x'][good] + x0,
pipeline['y'][good] + y0,
pipeline['fitResults_dx'][good],
pipeline['fitResults_dy'][good], spx,
spy)
from six.moves import cPickle
defFile = os.path.splitext(os.path.split(
self.visFr.GetTitle())[-1])[0] + '.sf'
fdialog = wx.FileDialog(
None,
'Save shift field as ...',
wildcard='Shift Field file (*.sf)|*.sf',
style=wx.FD_SAVE,
defaultDir=nameUtils.genShiftFieldDirectoryPath(),
defaultFile=defFile)
succ = fdialog.ShowModal()
if (succ == wx.ID_OK):
fpath = fdialog.GetPath()
#save as a pickle containing the data and voxelsize
fid = open(fpath, 'wb')
cPickle.dump((spx, spy), fid, 2)
fid.close()
def draw_line(self, x_y_axis):
X = randn(x_y_axis)
Y = randn(x_y_axis)
# it shows lines for random X and Y values
plt.scatter(X, Y, color='b')
plt.plot(X, Y)
plt.xlabel('X Axis')
plt.ylabel('Y Axis')
plt.title("Lines with scatter plot")
plt.show()
"""draw a scatter plot for random 500 x and y coordinates and style it"""
def plotants(self, timesteps=150, stepsize=0.03):
''' Plot the ants '''
pl.figure()
for t in range(timesteps):
pl.clf()
self.x += stepsize * pl.randn(self.numants)
self.y += stepsize * pl.randn(self.numants)
pl.scatter(self.x, self.y)
pl.xlim((-1, 1))
pl.ylim((-1, 1))
pl.title('t = %i / %i' % (t + 1, timesteps))
pl.pause(1e-3)
def plot_F_and_pi(F, pi, causes, title=""):
N, T, J = F.shape
pl.figure(figsize=(0.5 * T, 1.0 * J))
left = 2.0 / (T + 5.0)
right = 1 - 0.05 / T
bottom = 2.0 / (T + 5.0)
top = 1 - 0.05 / T
xmax = F.max()
dj = (top - bottom) / J
dt = (right - left) / T
ax = {}
for jj, j in enumerate(sorted(range(J), key=lambda j: pi[:, :, j].mean())):
for t in range(T):
pl.axes([left + t * dt, bottom + jj * dj, dt, dj])
pl.plot(pl.randn(N), F[:, t, j], "b.", alpha=0.5, zorder=-100)
pl.plot(0.5 * pl.randn(N), pi[:N, t, j], "g.", alpha=0.5, zorder=100)
# pi[:,t,j].sort()
# below = pi[:, t, j].mean() - pi[:,t,j][.025*N]
# above = pi[:,t,j][.975*N] - pi[:, t, j].mean()
# pl.errorbar([0], pi[:, t, j].mean(), [[below], [above]],
# fmt='gs', ms=10, mew=1, mec='white', linewidth=3, capsize=10,
# zorder=100)
pl.text(
-2.75,
xmax * 0.9,
"%.0f\n%.0f\n%.0f"
% (
100 * F[:, t, j].mean(),
100 * pi[:, t, j].mean(),
100 * pi[:, t, j].mean() - 100 * F[:, t, j].mean(),
),
va="top",
ha="left",
)
pl.xticks([])
pl.yticks([0.25, 0.5, 0.75])
if jj == 0:
pl.xlabel("%d\n%.0f" % (t + 1980, 100 * F[:, t, :].sum() / N))
if t > 0:
pl.yticks([])
else:
pl.ylabel(causes[j])
pl.axis([-3, 3, 0, xmax])
if title:
pl.figtext(0.01, 0.99, title, va="top", ha="left")
def test():
global a, v
print "nprocs", ocrofast.omp_nprocs()
print "nthreads", ocrofast.omp_nthreads()
from pylab import randn, newaxis, sum, argmin
a = randn(10000, 100)
v = randn(100)
t = timeit.timeit(lambda: argmindist(a, v), number=100)
result = argmindist(a, v)
print "ocrofast", result, "time", t
t = timeit.timeit(lambda: argmindist_py(a, v), number=100)
result = argmindist_py(a, v)
print "python", result, "time", t
def scatter_plot(self):
'''
Draw a scatter graph taking a random distribution in X and Y and
plotted against each other
'''
x_axis = randn(20) # random
y_axis = randn(20)
plt.scatter(x_axis, y_axis, color='r')
plt.title('Random distribution in X and Y')
plt.xlabel("X-axis")
plt.ylabel("Y_axis")
plt.savefig('data/4_1.scatter_plot.png')
plt.show()
def scatter_plot_empty_circle(self):
'''
Draw a scatter plot with empty circles taking a random distribution
in X and Y and plotted against each other
'''
x_axis = randn(20) # random
y_axis = randn(20)
plt.scatter(x_axis, y_axis, facecolors='none', edgecolors='r')
plt.title('Random distribution in X and Y')
plt.xlabel("X-axis")
plt.ylabel("Y_axis")
plt.savefig('data/4_1.scatter_plot.png')
plt.show()
def draw_scatter_plot(self, x_axis, y_axis):
X = randn(x_axis)
Y = randn(y_axis)
"""
Cartesian coordinates to display values for typically two variables for a set of data (x, y).
If the points are color-coded, one additional variable can be displayed (color = 'r')"""
plt.scatter(X, Y, color='r')
plt.xlabel("X axis")
plt.ylabel("Y axis")
plt.title("Scatter Plot with random number")
plt.show()
"""draw line and scatter plots for random 100 x and y coordinates"""
def example_histogram_3():
# first create a single histogram
mu, sigma = 200, 25
x = mu + sigma * plb.randn(10000)
plb.figure(6)
# finally: make a multiple-histogram of data-sets with different length
x0 = mu + sigma * plb.randn(10000)
x1 = mu + sigma * plb.randn(7000)
x2 = mu + sigma * plb.randn(3000)
n, bins, patches = plb.hist([x0, x1, x2], 10, histtype='bar')
plb.show()
def simulate(self, f_u, x0, tf):
"""
Simulate the system.
Parameters
----------
f_u: The input function f_u(t, x, i)
x0: The initial state.
tf: The final time.
Return
------
data : A StateSpaceDataArray object.
"""
#pylint: disable=too-many-locals, no-member
x0 = pl.matrix(x0)
assert x0.shape[1] == 1
t = 0
x = x0
dt = self.dt
data = StateSpaceDataList([], [], [], [])
i = 0
n_x = self.A.shape[0]
n_y = self.C.shape[0]
assert pl.matrix(f_u(0, x0, 0)).shape[1] == 1
assert pl.matrix(f_u(0, x0, 0)).shape[0] == n_y
# take square root of noise cov to prepare for noise sim
if pl.norm(self.Q) > 0:
sqrtQ = scipy.linalg.sqrtm(self.Q)
else:
sqrtQ = self.Q
if pl.norm(self.R) > 0:
sqrtR = scipy.linalg.sqrtm(self.R)
else:
sqrtR = self.R
# main simulation loop
while t + dt < tf:
u = f_u(t, x, i)
v = sqrtR.dot(pl.randn(n_y, 1))
y = self.measurement(x, u, v)
data.append(t, x, y, u)
w = sqrtQ.dot(pl.randn(n_x, 1))
x = self.dynamics(x, u, w)
t += dt
i += 1
return data.to_StateSpaceDataArray()
def computation(seed=0, n=1000):
# Make graph
pl.seed(int(seed))
fig = pl.figure()
ax = fig.add_subplot(111)
xdata = pl.randn(n)
ydata = pl.randn(n)
colors = sc.vectocolor(pl.sqrt(xdata**2+ydata**2))
ax.scatter(xdata, ydata, c=colors)
# Convert to FE
graphjson = sw.mpld3ify(fig, jsonify=False) # Convert to dict
return graphjson # Return the JSON representation of the Matplotlib figure
def estimate_vol_vol(self):
# dummy time series: must be replaced by a reader from an external source
rng = pd.date_range(start = '2010-01-01', end = '2011-01-01', freq='D')
tmp_0 = 100000. + 10000.*pl.randn(len(rng))
ts_volume = pd.Series(tmp_0, index=rng)
tmp_1 = 0.02 + 0.002*pl.randn(len(rng))
ts_volatility = pd.Series(tmp_1, index=rng)
# estimation of the daily volume and of the daily volatility as a flat average of the previuos n_days_mav
n_days_mav = 10
period_start = pd.to_datetime('2010-03-01') + pd.DateOffset(days=-(n_days_mav+1))
period_end = pd.to_datetime('2010-03-01') + pd.DateOffset(days=-1)
self.volume_est = ts_volume[period_start:period_end].mean()
self.volatility_est = ts_volatility[period_start:period_end].mean()
def genExampleAxisAligned():
# first choose a label
y = ''
feats = {}
if pylab.rand() < 0.5: # negative example
# from Nor([-1,0], 1)
y = '-1'
feats['x'] = pylab.randn() - 1
feats['y'] = pylab.randn()
else:
# from Nor([+1,0], 1)
y = '1'
feats['x'] = pylab.randn() + 1
feats['y'] = pylab.randn()
return (y, feats)
def matplotlib_plot(fname):
Xs, Ys = 6, 6
fig = pylab.figure(figsize=(Xs, Ys), frameon=False)
ax = fig.add_axes([0.0, 0.0, 1.0, 1.0], frameon=False)
ax.set_xticks([])
ax.set_yticks([])
N = 1000
x, y = pylab.randn(N), pylab.randn(N)
color = pylab.randn(N)
size = abs(400*pylab.randn(N))
p = pylab.scatter(x, y, c=color, s=size, alpha=0.75)
pylab.xlim(-2.0, 2.0)
pylab.ylim(-2.0, 2.0)
pylab.savefig(fname, dpi=200)
return Xs, Ys
def get_range(self):
""" Returns slant range to the object. Call once for each
new measurement at dt time from last call.
"""
# add some process noise to the system
vel = self.vel + 5*randn()
alt = self.alt + 10*randn()
self.pos = self.pos + vel*self.dt
# add measurment noise
err = self.pos * 0.05*randn()
slant_dist = sqrt(self.pos**2 + alt**2)
return slant_dist + err
def test_scatter_hist():
import pylab
X = pylab.randn(1000)
Y = pylab.randn(1000)
scatter_hist(X, Y)
df = pd.DataFrame({'X': X, 'Y': Y, 'size': X, 'color': Y})
scatter_hist(df, hist_position='left')
df = pd.DataFrame({'X': X})
try:
scatter_hist(df, hist_position='left')
assert False
except:
assert True
def scatter_plots(self):
print()
print(
"1. Draw a scatter graph taking a random distribution in X and Y and plotted against each other."
)
print("2. Exit")
print()
while True:
try:
print()
# accept choice from user
self.choice = input("Enter choice : ")
# validate choice number
valid_choice = validate_num(self.choice)
if valid_choice:
choice = int(self.choice)
if choice == 1:
x = input("Enter random distribution in x:")
# validate random value for x
validate = validate_num(x)
if validate:
x = int(x)
y = input("Enter random distribution in y:")
# validate random value for y
validate = validate_num(y)
if validate:
y = int(y)
# generate array of random number
x_value = randn(x)
y_value = randn(y)
# generate a scatter plots
plt.scatter(x_value, y_value, color='r')
plt.xlabel('X')
plt.ylabel('Y')
plt.show()
else:
print("Enter only numbers")
elif choice == 2:
exit()
else:
print("Enter valid choice")
else:
print("Enter only numbers")
except Exception as e:
print(e)
def test_plane_H():
if 1:
npts = 20
n = ut.normalized(np.array([1., 1, -1]))
d = 2.
X = pylab.randn(3, npts)
# X[:2,:]*n[:2] + d + n[2]*X[2] = 0 ==> X[2] = (-d - X[:2,:]*n[:2])/n[2]
X[2] = (-d - np.sum(X[:2] * n[:2, np.newaxis], axis=0)) / n[2]
assert np.all(np.abs(np.dot(X.T, n) + d) <= 0.001)
K1 = np.eye(3)
P1 = compose_P(K1, np.eye(3), np.zeros(3))
x1 = ut.homog_transform(P1, X)
K2 = 2 * np.eye(3)
R2 = np.eye(3)
t2 = np.array([0.5, 0, 0])
P2 = compose_P(K2, R2, t2)
x2 = ut.homog_transform(P2, X)
H = plane_H(n, d, K1, K2, R2, t2)
x2_est = ut.homog_transform(H, x1)
assert ut.a_eq(x2, x2_est)
if 1:
n = np.array([-0.09576725, -0.02749329, -0.995024])
d = 12.842613230422947
X = pylab.randn(3, npts)
X[2] = (-d - np.sum(X[:2] * n[:2, np.newaxis], axis=0)) / n[2]
# P1
K1 = np.array([[184.46153519, 0., 320.5], [0., -184.46153519, 240.5],
[0., 0., 1.]])
P1 = compose_P(K1, np.eye(3), np.zeros(3))
x1 = ut.homog_transform(P1, X)
# P2
K2 = np.array([[184.46153519, 0., 320.5], [0., -184.46153519, 240.5],
[0., 0., 1.]])
R2 = np.array([[0.99540027, -0.00263395, 0.09576725],
[0., 0.99962199, 0.02749329],
[-0.09580347, -0.02736683, 0.995024]])
t2 = np.array([-3.42297712, 6.86145016, -1.94439297])
P2 = compose_P(K2, R2, t2)
x2 = ut.homog_transform(P2, X)
H = plane_H(n, d, K1, K2, R2, t2)
x2_est = ut.homog_transform(H, x1)
assert ut.a_eq(x2, x2_est)
def generatePath(self, T):
"""
T: time period
r: rate of return
sigma: standard deviation
dt: time steps
drift: mean movement price
zn: array of random numbers with dimension(nPaths, nSteps)
"""
assert (T > 0), 'Time needs to be a positive number'
try:
S0 = self.initialPrice
r = self.rateOfReturn
sigma = self.stdev
nPaths = self.nPaths
nSteps = self.nSteps
dt = T/float(nSteps)
drift = r-0.5*(sigma**2)
zn = pylab.randn(nPaths, nSteps)
zn = np.vstack((zn, -zn))
S = pylab.zeros((nPaths, nSteps))
start = S0*pylab.ones((2*nPaths, 1))
next = S0*pylab.cumprod(pylab.exp(drift*dt +
sigma*pylab.sqrt(dt)*zn), 1)
except ValueError:
return 'Please check the value of the initial parameters.'
return pylab.hstack((start, next))
def __init__(self, numdims):
self.numdims = numdims
self.params = 0.001 * randn(numdims * 2)
self.wp = self.params[:numdims]
self.wm = self.params[numdims:]
self.b = 0.0
self.t = 1.0
def fig4():
# Figure 4
# histogram has the ability to plot multiple data in parallel ...
# Note the new color kwarg, used to override the default, which
# uses the line color cycle.
#
P.figure()
# create a new data-set
x = mu + sigma * P.randn(1000, 3)
n, bins, patches = P.hist(x,
10,
normed=1,
histtype='bar',
color=['crimson', 'burlywood', 'chartreuse'],
label=['Crimson', 'Burlywood', 'Chartreuse'])
P.legend()
#
# ... or we can stack the data
#
P.figure()
n, bins, patches = P.hist(x, 10, normed=1, histtype='bar', stacked=True)
P.show()
#
# we can also stack using the step histtype
#
P.figure()
n, bins, patches = P.hist(x, 10, histtype='step', stacked=True, fill=True)
def __init__(self,numin,numout):
self.numin = numin
self.numout = numout
self.params = 0.01 * randn(numin*numout+numout)
self.scorefunc = scorefunc.LinearRegressionScore(numin,numout,self.params)
self.scorefuncs = [self.scorefunc]
Contrastive.__init__(self,normalizeacrosscliques=False)
def __init__(self,numin,numclasses):
self.numin = numin
self.numclasses = numclasses
self.params = 0.01 * randn(self.numin*self.numclasses+self.numclasses)
self.scorefunc = logreg_score(self.numin,self.numclasses,self.params)
self.scorefuncs = [scorefunc]
Contrastive.__init__(self,normalizeacrosscliques=False)
def __init__(self, N, deg, fun, a, b, legendre):
self.inputs = []
self.labels = []
self.xrange_formated = []
if legendre == 1:
xrange = legendrePoints(N)
elif legendre == 2:
xrange = chebychevPoints(N)
else:
xrange = np.linspace(-1, 1, N)
for x in xrange: # Create the sample inputs and their labels
# y = np.random.randn()
y = truef(x, fun, a, b)
s = []
for i in np.arange(deg):
s.append(x**(i + 1))
self.inputs.append(s)
self.labels.append([y + pl.randn() / 30])
for x in np.linspace(-1.1, 1.1, 1000):
s = []
for i in np.arange(deg):
s.append(x**(i + 1))
self.xrange_formated.append(s)
self.len = N
def datasample(N, deg, fun, a, b, legendre):
inputs = []
labels = []
xrange_formated = []
if legendre == 1:
xrange = legendrePoints(N)
elif legendre == 2:
xrange = chebychevPoints(N)
else:
xrange = np.linspace(-1, 1, N)
for x in xrange: # Create the sample inputs and their labels
# y = np.random.randn()
y = truef(x, fun, a, b)
s = []
for i in np.arange(deg):
s.append(x**(i + 1))
inputs.append(s)
labels.append([y + pl.randn() / 30])
for x in np.linspace(-1.1, 1.1, 1000):
s = []
for i in np.arange(deg):
s.append(x**(i + 1))
xrange_formated.append(s)
return inputs, labels, xrange_formated
def synthbeats(duration, meanhr=60, stdhr=1, samplingfreq=250, sinfreq=None):
#Minimaly based on the parameters from:
#http://physionet.cps.unizar.es/physiotools/ecgsyn/Matlab/ecgsyn.m
#If freq exist it will be used to generate a sin instead of using rand
#Inputs: duration in seconds
#Returns: signal, peaks
t = np.arange(duration * samplingfreq) / float(samplingfreq)
signal = np.zeros(len(t))
print(len(t))
print(len(signal))
if sinfreq == None:
npeaks = 1.2 * (duration * meanhr / 60)
# add 20% more beats for some cummulative error
hr = pl.randn(int(npeaks)) * stdhr + meanhr
peaks = pl.cumsum(60. / hr) * samplingfreq
peaks = peaks.astype('int')
peaks = peaks[peaks < t[-1] * samplingfreq]
else:
hr = meanhr + sin(2 * pi * t * sinfreq) * float(stdhr)
index = int(60. / hr[0] * samplingfreq)
peaks = []
while index < len(t):
peaks += [index]
index += int(60. / hr[index] * samplingfreq)
signal[peaks] = 1.0
return t, signal, peaks
def generate_fe(out_fname='data.csv'):
""" Generate random data based on a fixed effects model
This function generates data for all countries in all regions, based on the model::
Y_r,c,t = beta * X_r,c,t
beta = [10., -.5, .1, .1, -.1, 0., 0., 0., 0., 0.]
X_r,c,t[0] = 1
X_r,c,t[1] = t - 1990.
X_r,c,t[k] ~ N(0, 1) for k >= 2
"""
c4 = countries_by_region()
a = 20.
beta = [10., -.5, .1, .1, -.1, 0., 0., 0., 0., 0.]
data = col_names()
for t in time_range:
for r in c4:
for c in c4[r]:
x = [1] + [t-1990.] + list(pl.randn(8))
y = float(pl.dot(beta, x))
se = 0.
data.append([r, c, t, a, y, se] + list(x))
write(data, out_fname)
def __init__(self,numstates, numdims):
self.numstates = numstates
self.numdims = numdims
self.numparams = self.numdims * self.numstates\
+ self.numdims**2 * self.numstates\
+ self.numstates\
+ self.numstates**2
self.params = zeros(self.numparams, dtype=float)
self.means = self.params[:self.numdims*self.numstates].reshape(
self.numdims,self.numstates)
self.covs = self.params[self.numdims * self.numstates:
self.numdims*self.numstates+
self.numdims**2 * self.numstates].reshape(
self.numdims, self.numdims, self.numstates)
self.logInitProbs = self.params[self.numdims * self.numstates +
self.numdims**2 * self.numstates:
self.numdims * self.numstates +
self.numdims**2 * self.numstates+
self.numstates]
self.logTransitionProbs = self.params[-self.numstates**2:].reshape(
self.numstates, self.numstates)
self.means[:] = 0.1 * randn(self.numdims, self.numstates)
for k in range(self.numstates):
self.covs[:,:,k] = eye(self.numdims) * 0.1;
self.logInitProbs[:] = log(ones(self.numstates, dtype=float) /
self.numstates)
self.logTransitionProbs[:] = log(ones((self.numstates,
self.numstates), dtype=float) /
self.numstates)
def specgram_demo(): ''' the demo in matplotlib. But calls interactive.specgram ''' from pylab import arange, sin, where, logical_and, randn, pi dt = 0.0005 t = arange(0.0, 20.0, dt) s1 = sin(2*pi*100*t) s2 = 2*sin(2*pi*400*t) # create a transient "chirp" mask = where(logical_and(t>10, t<12), 1.0, 0.0) s2 = s2 * mask # add some noise into the mix nse = 0.01*randn(len(t)) x = s1 + s2 + nse # the signal NFFT = 1024 # the length of the windowing segments Fs = int(1.0/dt) # the sampling frequency from ifigure.interactive import figure, specgram, nsec, plot, isec, clog, hold figure() hold(True) nsec(2) isec(0) plot(t, x) isec(1) specgram(x, NFFT=NFFT, Fs=Fs, noverlap=900) clog()
def synthbeats(duration, meanhr=60, stdhr=1, samplingfreq=250, sinfreq=None):
#Minimaly based on the parameters from:
#http://physionet.cps.unizar.es/physiotools/ecgsyn/Matlab/ecgsyn.m
#If frequ exist it will be used to generate a sin instead of using rand
#Inputs: duration in seconds
#Returns: signal, peaks
t = np.arange(duration * samplingfreq) / float(samplingfreq)
signal = np.zeros(len(t))
print(len(t))
print(len(signal))
if sinfreq == None:
npeaks = 1.2 * (duration * meanhr / 60)
# add 20% more beats for some cummulative error
hr = pl.randn(npeaks) * stdhr + meanhr
peaks = pl.cumsum(60. / hr) * samplingfreq
peaks = peaks.astype('int')
peaks = peaks[peaks < t[-1] * samplingfreq]
else:
hr = meanhr + sin(2 * pi * t * sinfreq) * float(stdhr)
index = int(60. / hr[0] * samplingfreq)
peaks = []
while index < len(t):
peaks += [index]
index += int(60. / hr[index] * samplingfreq)
signal[peaks] = 1.0
return t, signal, peaks
def test_speriodogram_2d():
data = randn(1024,2)
speriodogram(data)
data = np.array([marple_data, marple_data]).reshape(64,2)
speriodogram(data)
def _smooth_demo():
from numpy import linspace, sin, ones
from pylab import subplot, plot, hold, axis, legend, title, show, randn
t = linspace(-4, 4, 100)
x = sin(t)
xn = x + randn(len(t)) * 0.1
y = smooth(x)
ws = 31
subplot(211)
plot(ones(ws))
windows = ["flat", "hanning", "hamming", "bartlett", "blackman"]
hold(True)
for w in windows[1:]:
eval("plot(" + w + "(ws) )")
axis([0, 30, 0, 1.1])
legend(windows)
title("The smoothing windows")
subplot(212)
plot(x)
plot(xn)
for w in windows:
plot(smooth(xn, 10, w))
l = ["original signal", "signal with noise"]
l.extend(windows)
legend(l)
title("Smoothing a noisy signal")
show()
def __init__(self, q, hy, Y, reg, loo, learnbandwidth=False,
penfunc=penfunc_squarednorm,
pengrad=pengrad_squarednorm):
self.loo = loo #use leave-one-out-estimate?
self.learnbandwidth = learnbandwidth #optimize bandwidth, too?
self.q = q
self.Y = Y
self.penfunc = penfunc
self.pengrad = pengrad
self.loghy = array(log(hy))
self.d, self.numcases = Y.shape
self.lognumcases = log(self.numcases)
self.reg = reg #amount of regularization
self.Z = randn(self.q,self.numcases) * 0.1
#some quantities useful for gradient/cost computation:
GRAMY = dot(Y.T,Y)
self.YY = diag(GRAMY)[newaxis,:] + diag(GRAMY)[:,newaxis] - 2 * GRAMY
self.kY = -(1.0/exp(self.loghy)) * self.YY
if self.loo:
self.kY -= eye(self.numcases)*LARGE
self.KY = exp(self.kY) #kernel matrix
if not self.learnbandwidth:
self.params = self.Z.reshape(self.q*self.numcases)
else:
self.params = hstack((self.Z.reshape(self.q*self.numcases), self.loghy))
self.Z = self.params[:-1].reshape(self.q,self.numcases)
self.loghy = self.params[-1:]
def _pvoc(self, X_hat, Phi_hat=None, R=None):
"""
::
a phase vocoder - time-stretch
inputs:
X_hat - estimate of signal magnitude
[Phi_hat] - estimate of signal phase
[R] - resynthesis hop ratio
output:
updates self.X_hat with modified complex spectrum
"""
N = self.nfft
W = self.wfft
H = self.nhop
R = 1.0 if R is None else R
dphi = (2*P.pi * H * P.arange(N/2+1)) / N
print "Phase Vocoder Resynthesis...", N, W, H, R
A = P.angle(self.STFT) if Phi_hat is None else Phi_hat
phs = A[:,0]
self.X_hat = []
n_cols = X_hat.shape[1]
t = 0
while P.floor(t) < n_cols:
tf = t - P.floor(t)
idx = P.arange(2)+int(P.floor(t))
idx[1] = n_cols-1 if t >= n_cols-1 else idx[1]
Xh = X_hat[:,idx]
Xh = (1-tf)*Xh[:,0] + tf*Xh[:,1]
self.X_hat.append(Xh*P.exp( 1j * phs))
U = A[:,idx[1]] - A[:,idx[0]] - dphi
U = U - P.np.round(U/(2*P.pi))*2*P.pi
phs += (U + dphi)
t += P.randn()*P.sqrt(PVOC_VAR*R) + R # 10% variance
self.X_hat = P.np.array(self.X_hat).T
def spec_demo():
'''
the demo in matplotlib. But calls
interactive.specgram
'''
from pylab import arange, sin, where, logical_and, randn, pi
dt = 0.0005
t = arange(0.0, 20.0, dt)
s1 = sin(2 * pi * 100 * t)
s2 = 2 * sin(2 * pi * 400 * t)
# create a transient "chirp"
mask = where(logical_and(t > 10, t < 12), 1.0, 0.0)
s2 = s2 * mask
# add some noise into the mix
nse = 0.01 * randn(len(t))
x = s1 + s2 + nse # the signal
NFFT = 1024 # the length of the windowing segments
Fs = int(1.0 / dt) # the sampling frequency
from ifigure.interactive import figure, spec, nsec, plot, isec, clog, hold
figure()
hold(True)
nsec(2)
isec(0)
plot(t, x)
isec(1)
spec(t, x, NFFT=NFFT, noverlap=900)
clog()
def find_convex_hull(X, num_iter, num_points=None):
"""
if num_points is set to None, find_convex_hull will return all the points in
the convex hull (that have been found) sorted according to their sharpness.
Otherwise, it will return the N-sharpest points.
"""
(N, D) = X.shape
if (num_points == None):
num_points = N
# randomly choose 'num_iter' direction on the unit sphere.
# find the maximal point in the chosen direction, and add 1 to its counter.
# only points on the convex hull will be hit, and 'sharp' corners will
# have more hits than 'smooth' corners.
hits = p.zeros((N, 1))
for j in xrange(num_iter):
a = p.randn(D)
a = a / p.norm(a)
i = p.dot(X, a).argmax()
hits[i] += 1
# don't take points with 0 hits
num_points = min(num_points, sum(p.find(hits)))
# the indices of the n-best points
o = list(p.argsort(hits, 0)[xrange(-1, -(num_points+1), -1)].flat)
return X[o, :]
def __init__(self,numdims):
self.numdims = numdims
self.params = 0.001 * randn(numdims*2)
self.wp = self.params[:numdims]
self.wm = self.params[numdims:]
self.b = 0.0
self.t = 1.0
def fig4():
# Figure 4
# histogram has the ability to plot multiple data in parallel ...
# Note the new color kwarg, used to override the default, which
# uses the line color cycle.
#
P.figure()
# create a new data-set
x = mu + sigma * P.randn(1000, 3)
n, bins, patches = P.hist(x, 10, normed=1, histtype='bar',
color=['crimson', 'burlywood', 'chartreuse'],
label=['Crimson', 'Burlywood', 'Chartreuse'])
P.legend()
#
# ... or we can stack the data
#
P.figure()
n, bins, patches = P.hist(x, 10, normed=1, histtype='bar', stacked=True)
P.show()
#
# we can also stack using the step histtype
#
P.figure()
n, bins, patches = P.hist(x, 10, histtype='step', stacked=True, fill=True)
def make_distortions(size, distortions=[(5.0, 3)]):
"""Generate 2D distortions using filtered Gaussian noise.
The distortions are a sum of gaussian filtered white noise
with the given sigmas and maximum distortions.
:param size: size of the image for which distortions are generated
:param distortions: list of (sigma, maxdist) pairs
:returns: a grid of source coordinates suitable for scipy.ndimage.map_coordinates
"""
import pylab # FIXME
h, w = size
total = np.zeros((2, h, w), 'f')
for sigma, maxdist in distortions:
deltas = pylab.randn(2, h, w)
deltas = ndi.gaussian_filter(deltas, (0, sigma, 0))
deltas = ndi.gaussian_filter(deltas, (0, 0, sigma))
r = np.amax((deltas[..., 0]**2 + deltas[..., 1]**2)**.5)
deltas *= old_div(maxdist, r)
total += deltas
deltas = total
xy = np.array(np.meshgrid(list(range(h)),
list(range(w)))).transpose(0, 2, 1)
coords = deltas + xy
return coords
def test_scatter_hist():
import pylab
X = pylab.randn(1000)
Y = pylab.randn(1000)
scatter_hist(X, Y)
df = pd.DataFrame({"X": X, "Y": Y, "size": X, "color": Y})
scatter_hist(df, hist_position="left")
df = pd.DataFrame({"X": X})
try:
scatter_hist(df, hist_position="left")
assert False
except:
assert True
def fig3(x):
# Figure 3
# now we create a cumulative histogram of the data
#
P.figure()
n, bins, patches = P.hist(x, 50, normed=1, histtype='step', cumulative=True)
# add a line showing the expected distribution
y = P.normpdf(bins, mu, sigma).cumsum()
y /= y[-1]
l = P.plot(bins, y, 'k--', linewidth=1.5)
# create a second data-set with a smaller standard deviation
sigma2 = 15.
x = mu + sigma2 * P.randn(10000)
n, bins, patches = P.hist(x, bins=bins, normed=1, histtype='step', cumulative=True)
# add a line showing the expected distribution
y = P.normpdf(bins, mu, sigma2).cumsum()
y /= y[-1]
l = P.plot(bins, y, 'r--', linewidth=1.5)
# finally overplot a reverted cumulative histogram
n, bins, patches = P.hist(x, bins=bins, normed=1,
histtype='step', cumulative=-1)
P.grid(True)
P.ylim(0, 1.05)
def fBM(n, H):
"""
creates fractional Brownian motion
parameters: length of sample path;
Hurst exponent
this method uses another computational approach than fBM
http://en.wikipedia.org/wiki/Fractional_Brownian_motion#Method_2_of_simulation
I should somewhen look that up with proper references
- proper scaling is not implemented. Look up the article to implement it!
"""
gammaH = gamma(H + .5)
def KH(t, s):
"""
accordint to the article
"""
return (t - s)**(H - .5) / gammaH * hyp2f1(H - .5, .5 - H, H + .5,
1. - float(t) / float(s))
incs = randn(n + 1)
path = zeros(n + 1)
for pos in arange(n) + 1:
path[pos] = sum([KH(pos, x) * incs[x] for x in arange(pos) + 1])
return path[1:]
def fBM_nd(dims, H, return_mat=False, use_eig_ev=True):
"""
creates fractional Brownian motion
parameters: dims is a tuple of the shape of the sample path (nxd);
H: Hurst exponent
this is the slow version of fBM. It might, however, be more precise than
fBM, however - sometimes, the matrix square root has a problem, which might
induce inaccuracy
use_eig_ev: use eigenvalue decomposition for matrix square root computation
(faster)
"""
n = dims[0]
d = dims[1]
Gamma = zeros((n, n))
print('building ...\n')
for t in arange(n):
for s in arange(n):
Gamma[t, s] = .5 * ((s + 1)**(2. * H) +
(t + 1)**(2. * H) - abs(t - s)**(2. * H))
print('rooting ...\n')
if use_eig_ev:
ev, ew = eig(Gamma.real)
Sigma = dot(ew, dot(diag(sqrt(ev)), ew.T))
else:
Sigma = sqrtm(Gamma)
if return_mat:
return Sigma
v = randn(n, d)
return dot(Sigma, v)
def noise_distort1d(shape, sigma=100.0, magnitude=100.0):
h, w = shape
noise = ndi.gaussian_filter(pylab.randn(w), sigma)
noise *= magnitude / np.amax(abs(noise))
dys = np.array([noise] * h)
deltas = np.array([dys, np.zeros((h, w))])
return deltas
def main():
mu = pl.array([[0], [12], [24], [36]])
Sigma = pl.array([[3.01602775, 1.02746769, -3.60224613, -2.08792829],
[1.02746769, 5.65146472, -3.98616664, 0.48723704],
[-3.60224613, -3.98616664, 13.04508284, -1.59255406],
[-2.08792829, 0.48723704, -1.59255406, 8.28742469]])
# The data matrix is created for above mu and Sigma.
d, U = pl.eig(Sigma)
L = pl.diagflat(d)
A = pl.dot(U, pl.sqrt(L))
X = pl.randn(4, 1000)
# Y is the data matrix of random samples.
Y = pl.dot(A, X) + pl.tile(mu, 1000)
pl.figure(1)
pl.clf()
pl.plot(X[0], Y[1], '+', color='#0000FF', label='i=0,j=1')
pl.plot(X[0], Y[2], '+', color='#FF0000', label='i=0,j=2')
pl.plot(X[0], Y[3], '+', color='#00FF00', label='i=0,j=3')
pl.plot(X[1], Y[0], 'x', color='#FFFF00', label='i=1,j=0')
pl.plot(X[1], Y[2], 'x', color='#00FFFF', label='i=1,j=2')
pl.plot(X[1], Y[3], 'x', color='#444444', label='i=1,j=3')
pl.plot(X[2], Y[0], '.', color='#774411', label='i=2,j=0')
pl.plot(X[2], Y[1], '.', color='#222222', label='i=2,j=1')
pl.plot(X[2], Y[3], '.', color='#AAAAAA', label='i=2,j=3')
pl.plot(X[3], Y[0], '+', color='#FFAA22', label='i=3,j=0')
pl.plot(X[3], Y[1], '+', color='#22AAFF', label='i=3,j=1')
pl.plot(X[3], Y[2], '+', color='#FFDD00', label='i=3,j=2')
pl.legend()
pl.savefig('fig21.png')
def fBM_nd(dims, H, return_mat = False, use_eig_ev = True):
"""
creates fractional Brownian motion
parameters: dims is a tuple of the shape of the sample path (nxd);
H: Hurst exponent
this is the slow version of fBM. It might, however, be more precise than
fBM, however - sometimes, the matrix square root has a problem, which might
induce inaccuracy
use_eig_ev: use eigenvalue decomposition for matrix square root computation
(faster)
"""
n = dims[0]
d = dims[1]
Gamma = zeros((n,n))
print ('building ...\n')
for t in arange(n):
for s in arange(n):
Gamma[t,s] = .5*((s+1)**(2.*H) + (t+1)**(2.*H) - abs(t-s)**(2.*H))
print('rooting ...\n')
if use_eig_ev:
ev,ew = eig(Gamma.real)
Sigma = dot(ew, dot(diag(sqrt(ev)),ew.T) )
else:
Sigma = sqrtm(Gamma)
if return_mat:
return Sigma
v = randn(n,d)
return dot(Sigma,v)
def groupConfidenceWeight(AllData):
"""
weights answers by confidence for different groups
"""
# everybodygetup
subjects = range(len(AllData[1]['correct']))
distribution = np.array(py.zeros([20,len(subjects)]))
for i in subjects:
newdist = getIndConf(AllData, i)
print(len(newdist))
distribution[:,i] = newdist
m,n = py.shape(distribution)
for i in xrange(m):
for j in xrange(n):
distribution[i,j] = distribution[i,j] + py.randn(1)*0.05
print(distribution)
fig = py.figure()
ax20 = fig.add_subplot(111)
for i in xrange(n):
ax20.hist(distribution[:,i], bins=20, color='c', alpha=0.2,
edgecolor='none')
ax20.set_title('Weighted answers')
ax20.set_xlabel('Distribution')
ax20.set_ylabel('Counts')
def __init__(self, numstates, numdims):
self.numstates = numstates
self.numdims = numdims
self.numparams = self.numdims * self.numstates\
+ self.numdims**2 * self.numstates\
+ self.numstates\
+ self.numstates**2
self.params = zeros(self.numparams, dtype=float)
self.means = self.params[:self.numdims * self.numstates].reshape(
self.numdims, self.numstates)
self.covs = self.params[self.numdims *
self.numstates:self.numdims * self.numstates +
self.numdims**2 * self.numstates].reshape(
self.numdims, self.numdims, self.numstates)
self.logInitProbs = self.params[self.numdims * self.numstates +
self.numdims**2 *
self.numstates:self.numdims *
self.numstates +
self.numdims**2 * self.numstates +
self.numstates]
self.logTransitionProbs = self.params[-self.numstates**2:].reshape(
self.numstates, self.numstates)
self.means[:] = 0.1 * randn(self.numdims, self.numstates)
for k in range(self.numstates):
self.covs[:, :, k] = eye(self.numdims) * 0.1
self.logInitProbs[:] = log(
ones(self.numstates, dtype=float) / self.numstates)
self.logTransitionProbs[:] = log(
ones((self.numstates, self.numstates), dtype=float) /
self.numstates)
def plot_axes(fig):
# create some data to use for the plot
dt = 0.001
t = arange(0.0, 10.0, dt)
r = exp(-t[:1000]/0.05) # impulse response
x = randn(len(t))
s = convolve(x,r,mode=2)[:len(x)]*dt # colored noise
# the main axes is subplot(111) by default
axes = fig.gca()
axes.plot(t, s)
axes.set_xlim((0, 1))
axes.set_ylim((1.1*min(s), 2*max(s)))
axes.set_xlabel('time (s)')
axes.set_ylabel('current (nA)')
axes.set_title('Gaussian colored noise')
# this is an inset axes over the main axes
a = fig.add_axes([.65, .6, .2, .2], axisbg='y')
n, bins, patches = a.hist(s, 400, normed=1)
a.set_title('Probability')
a.set_xticks([])
a.set_yticks([])
# this is another inset axes over the main axes
a = fig.add_axes([.2, .6, .2, .2], axisbg='y')
a.plot(t[:len(r)], r)
a.set_title('Impulse response')
a.set_xlim((0, 0.2))
a.set_xticks([])
a.set_yticks([])
def binary_blur(image, sigma, noise=0.0):
p = percent_black(image)
blurred = ndi.gaussian_filter(image, sigma)
if noise > 0:
blurred += pylab.randn(*blurred.shape) * noise
t = percentile(blurred, p)
return array(blurred > t, 'f')
def _pvoc(self, X_hat, Phi_hat=None, R=None):
"""
::
a phase vocoder - time-stretch
inputs:
X_hat - estimate of signal magnitude
[Phi_hat] - estimate of signal phase
[R] - resynthesis hop ratio
output:
updates self.X_hat with modified complex spectrum
"""
N = self.nfft
W = self.wfft
H = self.nhop
R = 1.0 if R is None else R
dphi = (2 * P.pi * H * P.arange(N / 2 + 1)) / N
print("Phase Vocoder Resynthesis...", N, W, H, R)
A = P.angle(self.STFT) if Phi_hat is None else Phi_hat
phs = A[:, 0]
self.X_hat = []
n_cols = X_hat.shape[1]
t = 0
while P.floor(t) < n_cols:
tf = t - P.floor(t)
idx = P.arange(2) + int(P.floor(t))
idx[1] = n_cols - 1 if t >= n_cols - 1 else idx[1]
Xh = X_hat[:, idx]
Xh = (1 - tf) * Xh[:, 0] + tf * Xh[:, 1]
self.X_hat.append(Xh * P.exp(1j * phs))
U = A[:, idx[1]] - A[:, idx[0]] - dphi
U = U - P.np.round(U / (2 * P.pi)) * 2 * P.pi
phs += (U + dphi)
t += P.randn() * P.sqrt(PVOC_VAR * R) + R # 10% variance
self.X_hat = P.np.array(self.X_hat).T
def jitter(df, x, pct=.01):
""" Jitter column x by a certain percent
Results
-------
Return a pandas.Series with jittered values
"""
return df[x] + pl.randn(len(df.index)) * pct * df[x].mean()
def dW( self, t0, t1 ): """ Generate stochastic step for time interval t0..t1. Subclasses may override this term if something other than a Wiener process is needed. """ return randn(self.sdim) * sqrt(t1-t0)
