def kernel_caustic_masscalc(self,enc_r,enc_v,ENC_DIA_MASS,ENC_INF_MASS,ENC_DIA_CAU,ENC_INF_CAU,ENC_INF_NFW,enc_r200,enc_m200,enc_srad,enc_esrad,enc_vdisp,r_limit,vlimit,H0,q,k,root,beta,l=None):
enc_r,enc_v = array(enc_r),array(enc_v)
r = append(enc_r,enc_r)
v = append(enc_v,-1*enc_v)
xbeta,abeta = loadtxt(''+str(root)+'/nkern/Documents/MDB_milliMil_halodata/Caustic/average_betaprofile.tab',dtype='float',usecols=(0,1),unpack=True)
print ''
print '## Working on Ensemble #'+str(k)+''
if l != None:
print '## Line of Sight #'+str(l)+''
print '_______________________________________'
fit = polyfit((xbeta*enc_r200)[xbeta<4],abeta[xbeta<4],6)
res_array = array([1])
img_tot = 0
img_grad_tot = 0
img_inf_tot = 0
for u in xrange(res_array.size):
x_range,y_range,img,img_grad,img_inf = C.gaussian_kernel(r,v,enc_r200,normalization=H0,scale=q,res=200,adj=res_array[u],see=False)
img_tot += img/npmax(img)
img_grad_tot += img_grad/npmax(img_grad)
img_inf_tot += img_inf/npmax(img_inf)
## Define Beta ##
# beta = fit[0]*x_range**6+fit[1]*x_range**5+fit[2]*x_range**4+fit[3]*x_range**3+fit[4]*x_range**2+fit[5]*x_range+fit[6]
beta = np.zeros(x_range.size) + beta
## Caustic Surface Estimation ##
maxv = enc_r200*H0*sqrt(200)+500
## INFLECTION TECHNIQUE (MPHI) ##
Anew,threesig,dens_norm,e_dens_norm,srad,e_srad = C.level_search2(r,v,r,v,r,x_range,y_range,img_tot,img_inf_tot,H0*q,enc_r200,r_limit,maxv,beta,enc_srad,enc_esrad,use_vdisp=enc_vdisp,bin=k+1)
vdispersion = threesig/3.5
## DIAFERIO TECHNIQUE (CAUSTIC) ##
AnewD,threesigD,dens_normD,e_dens_normD,sradD,e_sradD = C.level_search(r,v,r,v,r,x_range,y_range,img_tot,H0*q,enc_r200,r_limit,maxv,beta,enc_srad,enc_esrad,use_vdisp=enc_vdisp,bin=k+1)
## Mass Calculation ##
# 1.) Using Diaferio Technique, pick out caustic, use caustic via massprofile to find mass, dia_caumass
# 2.) Using Inflection technique find caustic, fit to an nfw, find mass via dens_norm, inf_nfwmass
massprofile,integrand = C.masscalc(x_range,abs(C.Ar_final),enc_r200,enc_m200,vdispersion,beta=beta,conc=enc_r200/enc_srad)
massprofileD,integrandD = C.masscalc(x_range,abs(C.Ar_finalD),enc_r200,enc_m200,vdispersion,beta=beta,conc=enc_r200/enc_srad)
inf_nfwmass = 4*pi*dens_norm*(srad)**3*(log(1+enc_r200/srad)-enc_r200/srad/(1+enc_r200/srad))
dia_caumass = massprofileD[where(x_range[x_range >= 0] < enc_r200)[0][-1]]
ENC_DIA_MASS.append(dia_caumass)
ENC_INF_MASS.append(inf_nfwmass)
ENC_DIA_CAU.append(C.Ar_finalD)
ENC_INF_CAU.append(C.Ar_final)
ENC_INF_NFW.append(Anew)
return x_range,ENC_DIA_MASS,ENC_INF_MASS,ENC_DIA_CAU,ENC_INF_CAU,ENC_INF_NFW
def phi_f(mu, gam): """ Interpolate but extrapolate using the nearest value on the grid. """ mu = npmax(self.mumin, mu) gam = npmax(self.gammin, gam) mu = npmin(self.mumax, mu) gam = npmin(self.gammax, gam) return interp_phi(mu, gam)
def compare(frame):
"""Compare effects of depth isolation in histogram form"""
small = copy(frame)
done = __pipeline(frame)
done = done.astype(float) / npmax(done)
small[:5] = 0
small = (small.astype(float) / npmax(small))
plt.plot(done)
plt.plot(small)
plt.ylim(0, 1.0)
plt.show()
def Estimate_p0(self, x, y, p_0, Deblend_Check):
#INCLUDING P_0 HERE IS DIRTY BUT WE NEED IT FOR THE BATCH PROCESS LINE MEASURING
Emission_Line = True
#Check if emission or absorption
if self.LocalMedian > npmax(y):
Emission_Line = False
# print ' the line list is', self.List_BlendedLines[1][self.Current_BlendedGroup]
#Case of a blended line
if (Deblend_Check != None) and (p_0 == None):
if Emission_Line:
mu_0_List, A_0_List, Deblend_Check = self.FindMaxima(x, y, MinLevel = self.LocalMedian + 4 * self.SigmaContinuum, ListLines=self.List_BlendedLines[1][self.Current_BlendedGroup], Deblend_Check=Deblend_Check)
sigma_i = zeros(len(self.List_BlendedLines[0][self.Current_BlendedGroup]))
#Calculating the sigmas:
Interest_Points = [self.Selections[2], self.Selections[3]]
# for i in range(len(A_0_List) - 1):
#
# Peak_Left, Peak_Right = where(y == A_0_List[i])[0], where(y == A_0_List[i + 1])[0]
# x_BetweenMaxima, y_BetweenMaxima = x[Peak_Left : Peak_Right], y[Peak_Left: Peak_Right]
# x_m = x_BetweenMaxima[argmin(y_BetweenMaxima)]
# Interest_Points.append(x_m)
#
# Interest_Points.sort()
# for i in range(len(sigma_i)):
# sigma_i[i] = (Interest_Points[i+1] - Interest_Points[i]) /2 / 2.335 #If the lines are very closed no point in dividing by our /2 factor.
for i in range(len(sigma_i)):
sigma_i[i] = 1 #If the lines are very closed no point in dividing by our /2 factor.
p_0 = [A_0_List, mu_0_List, sigma_i]
if (Deblend_Check == None) and (p_0 == None):
self.Kmpfit_Dictionary = [{},{},{'limits':(0,3)}]
if Emission_Line:
A_0 = npmax(y)
mu_0 = x[where(y == A_0)]
sigma_0 = ((absolute(mu_0 - x[0]) + absolute(x[-1] - mu_0)) / 2.0) / 2.335
else:
#SHOULD WE CHANGE THIS ZEROLEV FOR THE ONE TRULLY AT A_0?
A_0 = 2 * self.LocalMedian - npmin(y)
mu_0 = x[where(y == npmin(y))]
sigma_0 = ((absolute(mu_0 - x[0]) + absolute(x[-1] - mu_0)) / 2.0) / 2.335
p_0 = [A_0, mu_0, sigma_0]
return Emission_Line, p_0, Deblend_Check
def plot_chiSq_Behaviour(self, Traces, labels):
n_traces = len(Traces)
self.FigConf(Figtype = 'Grid', n_colors = n_traces, n_columns=4, n_rows=2, FigHeight=9, FigWidth=16)
chisq_adapted = reshape(self.pymc_database.trace('ChiSq')[:], len(self.pymc_database.trace('ChiSq')[:])) * -2
y_lim = 30
min_chi_index = argmin(chisq_adapted)
for i in range(len(Traces)):
Trace = Traces[i]
label = labels[i]
if Trace != 'ChiSq':
self.Axis1[i].scatter(x = self.pymc_database.trace(Trace)[:], y = chisq_adapted, color=self.ColorVector[2][i])
x_min = npmin(self.pymc_database.trace(Trace)[:])
x_max = npmax(self.pymc_database.trace(Trace)[:])
self.Axis1[i].axvline(x = self.statistics_dict[Trace]['mean'], label = 'Inference value: ' + round_sig(self.statistics_dict[Trace]['mean'], 4,scien_notation=False), color='grey', linestyle = 'solid')
self.Axis1[i].scatter(self.pymc_database.trace(Trace)[:][min_chi_index], chisq_adapted[min_chi_index], color='Black', label = r'$\chi^{2}_{min}$ value: ' + round_sig(self.pymc_database.trace(Trace)[:][min_chi_index],4,scien_notation=False))
self.Axis1[i].set_ylabel(r'$\chi^{2}$',fontsize=20)
self.Axis1[i].set_ylim(0, y_lim)
self.Axis1[i].set_xlim(x_min, x_max)
self.Axis1[i].set_title(label,fontsize=20)
legend_i = self.Axis1[i].legend(loc='best', fontsize='x-large')
legend_i.get_frame().set_facecolor('white')
def plot(self, fig=None):
import matplotlib.pyplot as plt
if fig == None and not isinstance(self.fig, plt.Figure):
self.fig = plt.figure()
else:
self.fig = fig
# plt.gcf(self.fig)
plt.figure(self.fig.number)
plt.hold(True)
from numpy import max as npmax
mode = "logpow"
if mode == "logpow":
gdata = self.get_logpow()
plt.ylabel("power [db]")
plt.plot(self.get_xdata() / 1000., gdata)
ymax = int(npmax(gdata[1:]) / 10.) * 10 + 10
ymin = ymax - 80
plt.ylim(ymin, ymax)
plt.xlabel('Frequency [kHz]')
plt.locator_params(nbins=20, axis='x', tight=True)
# plt.locator_params(nbins=15, axis='y', tight=True, fontsize=1)
plt.grid()
plt.title(self.name)
return self
def _validate_seg(self):
if npmin(self.segments) < 0:
raise ValueError('Segments may only have positive integers')
if npmax(self.segments) >= len(self.vertices):
raise ValueError('Segments expects more vertices than provided')
self._validate_file_size('segments', self.segments)
self._validate_file_size('vertices', self.vertices)
return True
def general_max(*args):
if rank == 0:
if len(args)==1:
if isinstance(args[0],Raster) and args[0].data is not None:
return npmax(args[0].data)
else:
return max(*args)
else:
return 0
def validate(self):
"""Check if mesh content is built correctly"""
if npmin(self.triangles) < 0:
raise ValueError('Triangles may only have positive integers')
if npmax(self.triangles) >= len(self.vertices):
raise ValueError('Triangles expects more vertices than provided')
self._validate_file_size('vertices')
self._validate_file_size('triangles')
return True
def normalize(X):
"""
Normalize :math:`\mathbf{X}` to have values between 0 and 1.
:param X: (``numpy.ndarray``) Data matrix.
:return: (``numpy.ndarray``) Normalize matrix.
"""
return X / npmax(X)
def decorrelation_witer(W):
"""
Iterative MDUM decorrelation that avoids matrix inversion.
"""
lim = 1.0
tol = 1.0e-05
W = W/(W**2).sum()
while lim > tol:
W1 = (3.0/2.0)*W - 0.5*dot(dot(W,W.T),W)
lim = npmax(npabs(npabs(diag(dot(W1,W.T))) - 1.0))
W = W1
return W
def load_SpectraData(self):
self.Wave, self.Int, self.ExtraData = self.get_spectra_data(self.Current_Folder + self.Current_Spec)
Wmin = npmin(self.Wave)
Wmax = npmax(self.Wave)
#We erase the stored emision lines list
del self.EM_in_plot[:]
for i in range(len(self.Wavelengths_Total)):
if Wmin <= self.Wavelengths_Total[i] <= Wmax:
self.EM_in_plot.append(i)
def __pipeline(data):
data[:11] = 0
data[data < (npmax(data) / 10)] = 0
data = medfilt(data, 3)
intervals = __find_peak_intervals(data)
local_max = __get_local_maximums(data, intervals)
data = __remove_small_peaks(data, intervals, local_max, 10)
data = __thinout_peaks(data, intervals, local_max, 4)
intervals = __find_peak_intervals(data)
data = __remove_thin_peaks(data, intervals, 20)
data = __flat_binarize(data)
return data
def ica_par_fp(X, tolerance, g, gprime, orthog, alpha, maxIterations, Winit):
"""Parallel FastICA; orthog sets the method of unmixing vector decorrelation. This
fucntion is not meant to be directly called; it is wrapped by fastica()."""
n,p = X.shape
W = orthog(Winit)
lim = tolerance + 1
it = 1
while ((lim > tolerance) and (it < maxIterations)):
wtx = dot(W,X)
gwtx = g(wtx,alpha)
g_wtx = gprime(wtx,alpha)
W1 = dot(gwtx,X.T)/p - dot(diag(g_wtx.mean(axis=1)),W)
W1 = orthog(W1)
lim = npmax(npabs(npabs(diag(dot(W1,W.T))) - 1.0))
W = W1
it = it + 1
return W
def find_force_noise_levels(force_handle, finger, filter_order=3, filter_freq=0.3, measurements=10):
vector_filter=Filter_vector(order=filter_order, freq=filter_freq)
#stabilizing filter
for i in xrange(measurements):
force=force_handle.get_force(finger, update=True)
vector_filter.filter(force)
#real measurement
max_force=array([0.]*3)
max_norm_force=0.
for i in xrange(measurements):
force=force_handle.get_force(finger, update=True)
force_filtered=vector_filter.filter(force)
max_force=npmax(array([max_force, force_filtered]), axis=0)
norm_force=norm(force_filtered)
max_norm_force=max(max_norm_force, norm_force)
return(max_force, max_norm_force)
def Load_LineMesurerData(self):
#SHOULD THIS FOLDER BE LOCATED ON THE LINEMESURER... NOT SURE BECAUSE IT SHOULD BE INTEGRATED WITH THE INTERFACE...
self.Current_LinesLog = self.Current_Folder + self.Current_Code + '_'+ self.DataType + self.LinesLogExtension_Name
self.Wave, self.Int, self.ExtraData = self.File2Data(self.Current_Folder, self.Current_Spec)
Wmin = npmin(self.Wave)
Wmax = npmax(self.Wave)
#We erase the stored emision lines list
del self.EM_in_plot[:]
for i in range(len(self.Wavelengths_Total)):
if Wmin <= self.Wavelengths_Total[i] <= Wmax:
self.EM_in_plot.append(i)
if len(self.Labels_Total) < 3:
print "WARNING: Very few emission lines found within spectrum"
#Generate emision lines log
self.CleanTableMaker(self.Current_LinesLog, self.RemakeFiles, self.ColumnHeaderVector, self.ColumnWidth)
def Emission_Threshold(self, LineLoc, TotalWavelen, TotalInten):
#Use this method to determine the box and location of the emission lines
Bot = LineLoc - self.BoxSize
Top = LineLoc + self.BoxSize
indmin, indmax = searchsorted(TotalWavelen, (Bot, Top))
if indmax > (len(TotalWavelen)-1):
indmax = len(TotalWavelen)-1
PartialWavelength = TotalWavelen[indmin:indmax]
PartialIntensity = TotalInten[indmin:indmax]
Bot = LineLoc - 2
Top = LineLoc + 2
indmin, indmax = searchsorted(PartialWavelength, (Bot, Top))
LineHeight = npmax(PartialIntensity[indmin:indmax])
LineExpLoc = PartialWavelength[where(PartialIntensity == LineHeight)]
return PartialWavelength, PartialIntensity, LineHeight, LineExpLoc
def test_callback(samples, rtlsdr_obj):
#print(samples)
mpl.clf()
#dataset.extend(samples)
x = samples
yb1 = signal.convolve(x, filterB1)
#mpl.psd(yb1, NFFT=1024, Fc=0, Fs=48e3)
yn1 = downsample(yb1, 10)
#mpl.psd(yn1, NFFT=1024, Fc=0, Fs=sdr.rs/10)
zdis = discrim(yn1)
#mpl.psd(zdis, NFFT=1024, Fc=0, Fs=sdr.rs/10)
zb2 = signal.convolve(zdis, filterB2)
zn2 = downsample(zb2, 5)
mpl.psd(x)
mpl.psd(zn2, NFFT=1024, Fc=0, Fs=48e3)
mpl.plot(zdis)
mpl.pause(0.0001)
mpl.show(block=False)
zn2 /= npmax(abs(zn2),axis=0)
# complex2wav("teste.wav",48e3,zn2)
play(zn2, fs=48e3)
def res_rule_operator(self, phi):
"""
The reservation rule operator
-----------------------------
Qphi = c0*(1-beta) +
beta*integral( max{u(w'),phi(mu',gam')} * f(w'|mu,gam) )dw'
where:
u(w) = (1/a) * (1 - exp(-a*w))
f(w'|mu, gam) = N(mu, gam + gam_w)
gam' = 1/(1/gam + 1/gam_w)
mu' = gam' * (mu/gam + w'/gam_w)
The operator Q is a well-defined contraction mapping from
the complete metric space (b_kappa \Theta, rho_kappa) into
itself, where (b_kappa \Theta, rho_kappa) is the reweighted
space constructed by the weight function kappa.
Parameters
----------
phi : array_like(float, ndim=1, length=len(grid_points))
An approximate fixed point represented as a one-dimensional
array.
Returns
-------
new_phi : array_like(float, ndim=1, length=len(grid_points))
The updated fixed point.
"""
beta, gam_w, N_mc = self.beta, self.gam_w, self.N_mc
N_mc, a, c0 = self.N_mc, self.a, self.c0
draws = self.draws
u = self.u
interp_phi = LinearNDInterpolator(self.grid_points, phi)
def phi_f(mu, gam):
"""
Interpolate but extrapolate using the nearest value
on the grid.
"""
mu = npmax(self.mumin, mu)
gam = npmax(self.gammin, gam)
mu = npmin(self.mumax, mu)
gam = npmin(self.gammax, gam)
return interp_phi(mu, gam)
N = len(phi)
new_phi = np.empty(N)
for i in range(N):
mu, gam = self.grid_points[i, :]
# sample w' from f(w'|mu,gam) = N(mu, gam + gam_w)
draws_w = mu + np.sqrt(gam + gam_w) * draws
# the updated belief: mu', the update is based on
# the next period observation w'.
gam_prime = 1.0 / (1.0 / gam + 1.0 / gam_w) # a scalar
b1 = gam_prime / gam
mu_prime = b1 * mu + (1.0 - b1) * draws_w # an array with length N_mc
# the updated belief: gam'
gam_prime = gam_prime * np.ones(N_mc) # an array with length N_mc
expected_term = npmax(u(draws_w), phi_f(mu_prime, gam_prime))
expectation = np.mean(expected_term)
new_phi[i] = c0 * (1.0 - beta) + beta * expectation
return new_phi
# settings
lag = 15 # lag to be printed
ell_scale = 1.8 # ellipsoid radius coefficient
fit = 0 # no fit on marginals
dz_k_lim = [-1.99, 1.99] # lim for the axes
orange = [.9, .4, 0]
# autocorrelation test for invariance
f = figure(figsize=(12, 6))
InvarianceTestEllipsoid(dz_k,
acf_sign[0, 1:],
lag,
fit,
ell_scale,
bound=dz_k_lim)
# save_plot(ax=plt.gca(), extension='png', scriptname=os.path.basename('.')[:-3], count=plt.get_fignums()[-1])
# power low of autocorrelation decay
figure(figsize=(12, 6))
plot(ll, lcr[0, 1:], lw=1.5)
plot(ll, y, color=orange, lw=1.5)
plt.axis([min(ll), max(ll), min(lcr[0, 1:]), 0.95 * npmax(lcr[0, 1:])])
xlabel('ln(l)')
ylabel(
r'$\ln( | Cr(\Delta\tilde\zeta_{\kappa}, \Delta\tilde\zeta_{\kappa-l}) | )$'
)
legend(['empirical', 'linear fit\n $\lambda$ = % 1.3f' % -p[0]])
title('Autocorrelations decay: power law')
plt.show()
# save_plot(ax=plt.gca(), extension='png', scriptname=os.path.basename('.')[:-3], count=plt.get_fignums()[-1])
[CM, C] = ColorCodedFP(p, None, None, arange(0, 0.81, 0.01), 0, 1, [0.6, 0.2])
figure()
# colormap(CM)
# marginal of epsi_2 (change in 3yr yield)
ax = plt.subplot2grid((4, 5), (1, 0), rowspan=3, colspan=1)
plt.barh(x2[:-1], p2[0], height=x2[1] - x2[0], facecolor=grey,
edgecolor='k') # histogram
plot([0, 0], [mu_HBFP[1] - sdev[1], mu_HBFP[1] + sdev[1]], color=orange,
lw=5) # +/- standard deviation bar
plot(0,
epsi_out[1, -1],
color='b',
marker='o',
markerfacecolor='b',
markersize=5) # outlier
plt.ylim([npmin(epsi_out[1]), npmax(epsi_out[1])])
plt.xticks([])
plt.yticks([])
ax.invert_xaxis()
# marginal of epsi_1 (change in 1yr yield)
ax = plt.subplot2grid((4, 5), (0, 1), rowspan=1, colspan=4)
bar(x1[:-1], p1[0], width=x1[1] - x1[0], facecolor=grey, edgecolor='k')
plt.xticks([])
# # histogram
plot([mu_HBFP[0] - sdev[0], mu_HBFP[0] + sdev[0]], [0, 0], color=orange, lw=5)
# # +/- standard deviation bar
plot(epsi_out[0, -1],
0,
color='b',
marker='o',
markerfacecolor='b',
def plot_mag_series(times,
mags,
errs=None,
outfile=None,
sigclip=30.0,
timebin=None,
yrange=None):
'''This plots a magnitude time series.
If outfile is none, then plots to matplotlib interactive window. If outfile
is a string denoting a filename, uses that to write a png/eps/pdf figure.
timebin is either a float indicating binsize in seconds, or None indicating
no time-binning is required.
'''
if errs is not None:
# remove nans
find = npisfinite(times) & npisfinite(mags) & npisfinite(errs)
ftimes, fmags, ferrs = times[find], mags[find], errs[find]
# get the median and stdev = 1.483 x MAD
median_mag = npmedian(fmags)
stddev_mag = (npmedian(npabs(fmags - median_mag))) * 1.483
# sigclip next
if sigclip:
sigind = (npabs(fmags - median_mag)) < (sigclip * stddev_mag)
stimes = ftimes[sigind]
smags = fmags[sigind]
serrs = ferrs[sigind]
LOGINFO('sigclip = %s: before = %s observations, '
'after = %s observations' %
(sigclip, len(times), len(stimes)))
else:
stimes = ftimes
smags = fmags
serrs = ferrs
else:
# remove nans
find = npisfinite(times) & npisfinite(mags)
ftimes, fmags, ferrs = times[find], mags[find], None
# get the median and stdev = 1.483 x MAD
median_mag = npmedian(fmags)
stddev_mag = (npmedian(npabs(fmags - median_mag))) * 1.483
# sigclip next
if sigclip:
sigind = (npabs(fmags - median_mag)) < (sigclip * stddev_mag)
stimes = ftimes[sigind]
smags = fmags[sigind]
serrs = None
LOGINFO('sigclip = %s: before = %s observations, '
'after = %s observations' %
(sigclip, len(times), len(stimes)))
else:
stimes = ftimes
smags = fmags
serrs = None
# now we proceed to binning
if timebin and errs is not None:
binned = time_bin_magseries_with_errs(stimes, smags, serrs,
binsize=timebin)
btimes, bmags, berrs = (binned['binnedtimes'],
binned['binnedmags'],
binned['binnederrs'])
elif timebin and errs is None:
binned = time_bin_magseries(stimes, smags,
binsize=timebin)
btimes, bmags, berrs = binned['binnedtimes'], binned['binnedmags'], None
else:
btimes, bmags, berrs = stimes, smags, serrs
# finally, proceed with plotting
fig = plt.figure()
fig.set_size_inches(7.5,4.8)
plt.errorbar(btimes, bmags, fmt='go', yerr=berrs,
markersize=2.0, markeredgewidth=0.0, ecolor='grey',
capsize=0)
# make a grid
plt.grid(color='#a9a9a9',
alpha=0.9,
zorder=0,
linewidth=1.0,
linestyle=':')
# fix the ticks to use no offsets
plt.gca().get_yaxis().get_major_formatter().set_useOffset(False)
plt.gca().get_xaxis().get_major_formatter().set_useOffset(False)
# get the yrange
if yrange and isinstance(yrange,list) and len(yrange) == 2:
ymin, ymax = yrange
else:
ymin, ymax = plt.ylim()
plt.ylim(ymax,ymin)
plt.xlim(npmin(btimes) - 0.001*npmin(btimes),
npmax(btimes) + 0.001*npmin(btimes))
plt.xlabel('time [JD]')
plt.ylabel('magnitude')
if outfile and isinstance(outfile, str):
plt.savefig(outfile,bbox_inches='tight')
plt.close()
return os.path.abspath(outfile)
else:
plt.show()
plt.close()
return
# ## Compute the risk neutral probabilities using the Stochastic Discount Factors found at the previous steps
q_true = SDF_true.T @ np.diagflat(p) / v_tnow[0]
q_kern = SDF_kern @ np.diagflat(p) / v_tnow[0]
q_mre = SDF_mre @ np.diagflat(p) / v_tnow[0]
# ## For each instrument in the market and for each risk neutral probability found at the previous step, compute the left-hand side and the right-hand side of the fundamental theorem of asset pricing
y = v_tnow / v_tnow[0]
x = r_['-1', V_payoff @ q_true.T, V_payoff @ q_kern.T, V_payoff @ q_mre.T]
# ## Generate the figure
pick = range(50) # We just pick first 50 dots to make the figure more
figure()
plot([npmin(y[pick]), npmax(y[pick])], [npmin(y[pick]), npmax(y[pick])], lw=1)
scatter(np.array(y[pick]),
np.array(x[pick, 0]),
marker='x',
s=50,
color=[1, 0.3, 0],
lw=1)
scatter(np.array(y[pick]),
np.array(x[pick, 1]),
marker='o',
s=70,
color=[0.4, 0.4, 0],
facecolor="none")
scatter(np.array(y[pick]),
np.array(x[pick, 2]),
marker='.',
def self_stack_kernel_caustic_masscalc(self,ENC_R,ENC_V,ENC_CAUMASS,ENC_INFMASS,ENC_CAUSURF,ENC_INFSURF,ENC_INFNFW,ENC_R200,ENC_M200,ENC_SRAD,ENC_ESRAD,ENC_VDISP,r_limit,vlimit,H0,q,k,root,beta,l=None,samp_size=None):
ENC_R,ENC_V = array(ENC_R),array(ENC_V)
R = append(ENC_R,ENC_R)
V = append(ENC_V,-1*ENC_V)
xbeta,abeta = loadtxt(''+str(root)+'/nkern/Documents/MDB_milliMil_halodata/Caustic/average_betaprofile.tab',dtype='float',usecols=(0,1),unpack=True)
print ''
print '## Working on Cluster #'+str(k)+''
if l != None:
print '## Line of Sight #'+str(l)+''
print '_______________________________________'
print 'galaxies within r200 =',where(ENC_R<ENC_R200)[0].size
if samp_size != None:
print 'sample size =',samp_size
### Kernel Density Estimation ###
fit = polyfit((xbeta*ENC_R200)[xbeta<4],abeta[xbeta<4],6)
res_array = array([1])
img_tot = 0
img_grad_tot = 0
img_inf_tot = 0
for u in xrange(res_array.size):
x_range,y_range,img,img_grad,img_inf = C.gaussian_kernel(R,V,ENC_R200,normalization=H0,scale=q,res=200,adj=res_array[u],see=False)
img_tot += img/npmax(img)
img_grad_tot += img_grad/npmax(img_grad)
img_inf_tot += img_inf/npmax(img_inf)
### Define Beta ###
# beta = fit[0]*x_range**6+fit[1]*x_range**5+fit[2]*x_range**4+fit[3]*x_range**3+fit[4]*x_range**2+fit[5]*x_range+fit[6]
beta = np.zeros(x_range.size) + beta
### Caustic Surface Estimation ###
maxv = ENC_R200*H0*sqrt(200)+500
# Inflection Technique to find Surface
Anew,threesig,dens_norm,e_dens_norm,srad,e_srad = C.level_search2(R,V,R,V,R,x_range,y_range,img_tot,img_inf_tot,H0*q,ENC_R200,r_limit,maxv,beta,ENC_SRAD,ENC_ESRAD,use_vdisp=ENC_VDISP,bin=k+1)
vdispersion = threesig/3.5
# Caustic Technique to find Surface
AnewD,threesigD,dens_normD,e_dens_normD,sradD,e_sradD = C.level_search(R,V,R,V,R,x_range,y_range,img_tot,H0*q,ENC_R200,r_limit,maxv,beta,ENC_SRAD,ENC_ESRAD,use_vdisp=ENC_VDISP,bin=k+1)
### Mass Calculation ###
# Normal "Diaferio 1999 Mass" with fbeta = .65 (inherent setting is fbeta = .5)
massprofile,integrand = C.masscalc(x_range,abs(C.Ar_finalD),ENC_R200,ENC_M200,vdispersion,beta=beta,conc=ENC_R200/ENC_SRAD,dstyle=True)
cau_diamass = (.65/.5) * massprofile[where(x_range[x_range >= 0] < ENC_R200)[0][-1]]
# Caustic Mass with Concentration parameter, no fbeta
massprofile2,integrand2 = C.masscalc(x_range,abs(C.Ar_finalD),ENC_R200,ENC_M200,vdispersion,beta=beta,conc=ENC_R200/ENC_SRAD)
cau_concmass = massprofile2[where(x_range[x_range >= 0] < ENC_R200)[0][-1]]
# Inflection Mass using NFW
inf_nfwmass = 4*pi*dens_norm*(srad)**3*(log(1+ENC_R200/srad)-ENC_R200/srad/(1+ENC_R200/srad))
# Labeling Caustic Mass as Diaferio 1999 fbeta mass, in future, Caustic Mass is concentration mass
ENC_CAUMASS.append(cau_diamass)
ENC_INFMASS.append(inf_nfwmass)
ENC_CAUSURF.append(C.Ar_finalD)
ENC_INFSURF.append(C.Ar_final)
ENC_INFNFW.append(Anew)
return x_range,ENC_CAUMASS,ENC_INFMASS,ENC_CAUSURF,ENC_INFSURF,ENC_INFNFW
# y_hor = t.pdf('tlocationscale', x_hor, m_tau, sigma_tau, 1)
y_hor = t.pdf((x_hor - m_tau) / sigma_tau, 1) / sigma_tau
# -
# ## Create figure
# +
s_ = 2 # number of plotted observation before projecting time
m = min([
npmin(X),
npmin(x[0, t_ - s_:t_]),
npmin([x[0, t_ - 1], m_tau]) - 6 * sigma_tau
])
M = max([
npmax(X),
npmax(x[0, t_ - s_:t_]),
npmax([x[0, t_ - 1], m_tau]) + 6 * sigma_tau
])
t = arange(-s_, tau + 1)
max_scale = tau / 4
scale = max_scale / max(y_hor)
# preliminary computations
tau_red = arange(0, tau, 0.1).reshape(1, -1)
m_red = x[0, t_ - 1] + mu * tau_red
sigma_red = sigma * tau_red
redline1 = m_red + 2 * sigma_red
redline2 = m_red - 2 * sigma_red
f = figure()
# foreign exchange rate figure
figure()
# simulated path, mean and standard deviation
plot(horiz_u[:i + 1], FX[:j_sel, :i + 1].T, color=lgrey, lw=1)
xticks(arange(0, t_end + 1, 20))
xlim([min(horiz_u), max(horiz_u) + 1])
l1 = plot(horiz_u[:i + 1], Mu_FX[:i + 1], color='g')
l2 = plot(horiz_u[:i + 1], Mu_FX[:i + 1] + Sigma_FX[:i + 1], color='r')
plot(horiz_u[:i + 1], Mu_FX[:i + 1] - Sigma_FX[:i + 1], color='r')
# histogram
option = namedtuple('option', 'n_bins')
option.n_bins = round(10 * log(j_))
y_hist, x_hist = HistogramFP(FX[:, [i]].T, flex_probs_scenarios.T, option)
scale = 1500 * Sigma_FX[i] / npmax(y_hist)
y_hist = y_hist * scale
shift_y_hist = horiz_u[i] + y_hist
# empirical pdf
emp_pdf = plt.barh(x_hist[:-1],
shift_y_hist[0] - horiz_u[i],
height=x_hist[1] - x_hist[0],
left=horiz_u[i],
facecolor=lgrey,
edgecolor=lgrey)
plot(shift_y_hist[0], x_hist[:-1], color=dgrey, lw=1) # border
legend(handles=[l1[0], l2[0], emp_pdf[0]],
labels=['mean', ' + / - st.deviation', 'horizon pdf'])
xlabel('time (days)')
def kernel_caustic_masscalc(self,ENC_R,ENC_V,ENC_M200,ENC_R200,ENC_SRAD,ENC_ESRAD,ENC_HVD,halo_num,bin_range,gal_num,H0,q,r_limit,run_num,use_mems):
#Dummy arrays that Caustic functions call but have no use..
potential = ones(halo_num/bin_range)
ENC_GVD = ones(halo_num/bin_range)
Rt=[] # Doubling 3d data to mimic projected data, need for levelsearch2()
Vt=[]
for j in range(len(ENC_R)):
Rt.append(append(ENC_R[j],ENC_R[j]))
Vt.append(append(ENC_V[j],-ENC_V[j]))
ENC_R = array(Rt)
ENC_V = array(Vt)
xbeta,abeta = loadtxt('/n/Christoq1/nkern/Documents/MDB_milliMil_halodata/Caustic/average_betaprofile.tab',dtype='float',usecols=(0,1),unpack=True)
ENC_INF_MPROF = []
ENC_INF_NFW = []
ENC_INF_CAU = []
ENC_DIA_MPROF = []
ENC_DIA_NFW = []
ENC_DIA_CAU = []
ENC_DIA_CAUMASS = [] #Diaferio technique to produce Caustic, Caustic to find mass
ENC_INF_CAUMASS = [] #Inflection technique to produce Casutic, Caustic to find mass
ENC_DIA_NFWMASS = [] #Diaferio technique to produce Caustic, NFW fit to find mass....?? check w/ dan on levelsearch() nfw fit, double parameter???
ENC_INF_NFWMASS = [] #Inflection technique to produce Caustic, NFW fit to find mass
### Kernel Density Estimation ###
for k in range((run_num[1]-run_num[0])/bin_range):
''' k refers to the halo's position in ENC_* arrays, run_num/bin_range'''
print ''
print 'WORKING ON ENSEMBLE CLUSTER','#'+str(k)+''
fit = polyfit((xbeta*ENC_R200[k])[xbeta<4],abeta[xbeta<4],6)
res_array = array([1])
img_tot = 0
img_grad_tot = 0
img_inf_tot = 0
for u in range(res_array.size):
x_range,y_range,img,img_grad,img_inf = C.gaussian_kernel(ENC_R[k],ENC_V[k],ENC_R200[k],normalization=H0,scale=q,res=200,adj = res_array[u],see=False)
img_tot += img/npmax(img)
img_grad_tot += img_grad/npmax(img_grad)
img_inf_tot += img_inf/npmax(img_inf)
### Define Beta ###
beta = fit[0]*x_range**6 + fit[1]*x_range**5 + fit[2]*x_range**4 + fit[3]*x_range**3 + fit[4]*x_range**2 + fit[5]*x_range + fit[6]
### Caustic Surface Estimation ###
maxv = ENC_R200[k]*H0*sqrt(200)+500
## INFLECTION TECHNIQUE (M_PHI) ##
Anew,threesig,dens_norm,e_dens_norm,srad,e_srad,Ar_final = C.level_search2(ENC_R[k],ENC_V[k],ENC_R[k],ENC_V[k],ENC_R[k],x_range,y_range,img_tot,img_inf_tot,H0*q,ENC_R200[k],r_limit,maxv,beta,potential,ENC_SRAD[k],ENC_ESRAD[k],use_vdisp=ENC_HVD[k],use_mems=use_mems,bin=k)
vdispersion = threesig/3.5
## DIAFERIO TECHNIQUE (CAUSTIC) ##
Anewd,threesigd,dens_normd,e_dens_normd,sradd,e_sradd,Ar_finald = C.level_search(ENC_R[k],ENC_V[k],ENC_R[k],ENC_V[k],ENC_R[k],x_range,y_range,img_tot,H0*q,ENC_R200[k],r_limit,maxv,beta,potential,ENC_SRAD[k],ENC_ESRAD[k],use_vdisp=ENC_HVD[k],use_mems=use_mems,bin=k)
vdispersiond = threesigd/3.5
### Mass Calculation ###
# Ways to calculate mass:
# 1.) Using Inflection technique, pick out caustic, use caustic to calculate mass via massprofile
# 2.) Using Diaferio technique, pick out casutic, use caustic to calculate mass via massprofiled
# 3.) Using Inflection Causitc, fit an NFW, use NFW to calculate mass via dens_norm
# 4.) Using Diaferio technique, fit an NFW, use NFW to calculate mass via dens_normd
massprofile,integrand = C.masscalc(x_range,abs(Ar_final),ENC_R200[k],ENC_M200[k],vdispersion,beta,conc=ENC_R200[k]/ENC_SRAD[k])
massprofiled,integrandd = C.masscalc(x_range,abs(Ar_finald),ENC_R200[k],ENC_M200[k],vdispersiond,beta,conc=ENC_R200[k]/ENC_SRAD[k])
inf_nfwmass = 4*pi*dens_norm*(srad)**3*(np.log(1+ENC_R200[k]/srad)-ENC_R200[k]/srad/(1+ENC_R200[k]/srad))
dia_nfwmass = 4*pi*dens_normd*(sradd)**3*(np.log(1+ENC_R200[k]/sradd)-ENC_R200[k]/sradd/(1+ENC_R200[k]/sradd))
inf_caumass = massprofile[where(x_range[x_range >= 0] < ENC_R200[k])[0][-1]]
dia_caumass = massprofiled[where(x_range[x_range >= 0] < ENC_R200[k])[0][-1]]
### Data Appending ###
ENC_INF_NFWMASS.append(inf_nfwmass)
ENC_INF_CAUMASS.append(inf_caumass)
ENC_DIA_NFWMASS.append(dia_nfwmass)
ENC_DIA_CAUMASS.append(dia_caumass)
ENC_INF_MPROF.append(massprofile)
ENC_INF_NFW.append(Anew)
ENC_INF_CAU.append(Ar_final)
ENC_DIA_MPROF.append(massprofiled)
ENC_DIA_NFW.append(Anewd)
ENC_DIA_CAU.append(Ar_finald)
ENC_INF_MPROF,ENC_DIA_MPROF = array(ENC_INF_MPROF),array(ENC_DIA_MPROF)
ENC_INF_NFW,ENC_DIA_NFW = array(ENC_INF_NFW),array(ENC_DIA_NFW)
ENC_INF_CAU,ENC_DIA_CAU = array(ENC_INF_CAU),array(ENC_DIA_CAU)
return x_range,ENC_INF_NFWMASS,ENC_DIA_NFWMASS,ENC_INF_CAUMASS,ENC_DIA_CAUMASS,ENC_INF_MPROF,ENC_INF_NFW,ENC_INF_CAU,ENC_DIA_MPROF,ENC_DIA_NFW,ENC_DIA_CAU
def _settling_flux(X, v_max, v_max_practical, X_min, rh, rp, n0):
X_star = npmax(X-X_min, n0)
v = npmin(v_max_practical, v_max*(npexp(-rh*X_star) - npexp(-rp*X_star)))
return X*npmax(v, n0)
def integrand(x):
"Integral expression on right-hand side of operator"
return npmax(x, phi_f(q(x, pi))) * (pi*f(x) + (1 - pi)*g(x))
def garch1f4(x, eps, df):
## Fit a GARCH(1,1) model with student-t errors
# INPUTS
# x : [vector] (T x 1) data generated by a GARCH(1,1) process
# OPS
# q : [vector] (4 x 1) parameters of the GARCH(1,1) process
# qerr : [vector] (4 x 1) standard error of parameter estimates
# hf : [scalar] current conditional heteroskedasticity estimate
# hferr : [scalar] standard error on hf
# NOTE
# o Uses a conditional t-distribution with fixed degrees of freedom
# o Originally written by Olivier Ledoit, 4/28/1997
# o Difference with garch1f: errors come from the score alone
# Parameters
gold = (1 + sqrt(5)) / 2 # step size increment
tol1 = 1e-7 # for termination criterion
tol2 = 1e-7 # for closeness to boundary
big = 2 # for making the hessian negative definite
maxiter = 50 # maximum number of iterations
n = 30 # number of points on the grid
# Rescale
y = (x.flatten() - mean(x.flatten()))**2
t = len(y)
scale = sqrt(mean(y**2))
y = y / scale
s = mean(y)
# Grid search
[ag, bg] = meshgrid(linspace(0, 1 - eps, n), linspace(0, 1 - eps, n))
cg = np.maximum(s * (1 - ag - bg), 0)
likeg = -np.Inf * ones((n, n))
for i in range(n):
for j in range(n - i):
h = filter(array([0, ag[i, j]]), array([1, -bg[i, j]]), y * (df - 2) / df, zi=array([s * (df - 2) / df]))[0] \
+ filter(array([0, cg[i, j]]), array([1, -bg[i, j]]), ones(t))
likeg[i, j] = -npsum(log(h) + (df + 1) * log(1 + y / h / df))
maxlikeg = npmax(likeg)
maxima = where(likeg == maxlikeg) ##ok<MXFND>
# Initialize optimization
a = r_[cg[maxima], ag[maxima], bg[maxima]]
best = 0
da = 0
# term = 1
# negdef = 0
iter = 0
# Begin optimization loop
while iter < maxiter:
iter = iter + 1
# New parameter1
a = a + gold**best * da
# Conditional variance
h = filter([0, a[1]], [1, -a[2]], y * (df - 2) / df, zi=array([s * (df - 2) / df]))[0] \
+ filter([0, a[0]], [1, -a[2]], ones(t))
# Likelihood
if (any(a < 0) or ((a[1] + a[2]) > 1 - eps)):
like = -np.Inf
else:
like = -npsum(log(h) + (df + 1) * log(1 + y / h / df))
# Gradient
GG = r_['-1',
filter([0, 1], [1, -a[2]], ones(t))[..., newaxis],
filter([0, 1], [1, -a[2]],
y * (df - 2) / df)[..., newaxis],
filter([0, 1], [1, -a[2]], h)[..., newaxis]]
g1 = ((df + 1) * (y / (y + df * h)) - 1) / h
G = GG * repeat(g1.reshape(-1, 1), 3, axis=1)
gra = npsum(G, axis=0)
# Hessian
GG2 = GG[:,
[0, 1, 2, 0, 1, 2, 0, 1, 2]] * GG[:,
[0, 0, 0, 1, 1, 1, 2, 2, 2]]
g2 = -((df + 1) *
(y /
(y + df * h)) - 1) / h**2 - (df *
(df + 1)) * (y /
(y + df * h)**2 / h)
HH = zeros((t, 9))
HH[:, 2] = filter([0, 1], [1, -a[2]], GG[:, 0])
HH[:, 6] = HH[:, 2]
HH[:, 5] = filter([0, 1], [1, -a[2]], GG[:, 1])
HH[:, 7] = HH[:, 5]
HH[:, 8] = filter([0, 2], [1, -a[2]], GG[:, 2])
H = GG2 * repeat(g2.reshape(-1, 1), 9, axis=1) + HH * repeat(
g1.reshape(-1, 1), 9, axis=1)
hes = reshape(npsum(H, axis=0), (3, 3), 'F')
# Negative definite
d, u = eig(hes)
# d = diagflat(d)
if any(d > 0):
negdef = 0
d = min(d, max(d[d < 0]) / big)
hes = u @ diagflat(d) @ u.T
else:
negdef = 1
# Direction
da = -gra.dot(pinv(hes))
# Termination criterion
term = da @ gra.T
if (term < tol1) and negdef:
break
# Step search
best = 0
newa = a + gold**(best - 1) * da
if (any(newa < 0) or (newa[1] + newa[2] > 1 - eps)):
left = -np.Inf
else:
h = filter([0, newa[1]], [1, -newa[2]], y * (df - 2) / df, zi=array([s * (df - 2) / df]))[0] \
+ filter([0, newa[0]], [1, -newa[2]], ones(t))
left = -sum(log(h) + (df + 1) * log(1 + y / h / df))
newa = a + gold**best * da
if (any(newa < 0) or (newa[1] + newa[2] > 1 - eps)):
center = -np.Inf
else:
h = filter([0, newa[1]], [1, -newa[2]], y * (df - 2) / df, zi=array([s * (df - 2) / df]))[0] \
+ filter([0, newa[0]], [1, -newa[2]], ones(t))
center = -sum(log(h) + (df + 1) * log(1 + y / h / df))
newa = a + gold**(best + 1) * da
if (any(newa < 0) or (newa[1] + newa[2] > 1 - eps)):
right = -np.Inf
else:
h = filter([0, newa[1]], [1, -newa[2]], y * (df - 2) / df, zi=array([s * (df - 2) / df]))[0] \
+ filter([0, newa[0]], [1, -newa[2]], ones(t))
right = -sum(log(h) + (df + 1) * log(1 + y / h / df))
if all(like > array([left, center, right])) or all(
left > array([center, right])):
while True:
best = best - 1
center = left
newa = a + gold**(best - 1) * da
if (any(newa < 0) or (newa[1] + newa[2] > 1 - eps)):
left = -np.Inf
else:
h = filter([0, newa[1]], [1, -newa[2]], y * (df - 2) / df, zi=array([s * (df - 2) / df]))[0] \
+ filter([0, newa[0]], [1, -newa[2]], ones(t))
left = -sum(log(h) + (df + 1) * log(1 + y / h / df))
if all(center >= array([like, left])):
break
elif all(right > array([left, center])):
while True:
best = best + 1
center = right
newa = a + gold**(best + 1) * da
if (any(newa < 0) or (newa[1] + newa[2]) > 1 - eps):
right = -np.Inf
else:
h = filter([0, newa[1]], [1, -newa[2]], y * (df - 2) / df, zi=array([s * (df - 2) / df]))[0] \
+ filter([0, newa[0]], [1, -newa[2]], ones(t))
right = -npsum(log(h) + (df + 1) * log(1 + y / h / df))
if center > right:
break
# If stuck at boundary then stop
if (center == like) and (any(a < tol2) or (a[1] + a[2]) > 1 - tol2):
break
# End of optimization loop
a[a < tol2] = zeros(len(a[a < tol2]))
if a[1] + a[2] > 1 - tol2:
if a[1] < 1 - tol2:
a[1] = a[1] + (1 - a[1] - a[2])
else:
a[2] = a[2] + (1 - a[1] - a[2])
# Estimation error and volatility forecast
# aerr=inv(G.T@G)
tmp = (G.T @ G)
aerr = tmp.dot(pinv(eye(tmp.shape[0])))
hf = a[0] + a[1] * y[t - 1] * (df - 2) / df + a[2] * h[t - 1]
gf = r_[1, y[t - 1], h[t - 1]] + a[2] * GG[t - 1, :]
hferr = gf @ aerr @ gf.T
aerr = diagflat(aerr).T
# Revert to original scale
a[0] = a[0] * scale
aerr[0] = aerr[0] * scale**2
hf = hf * scale
hferr = hferr * scale**2
aerr = sqrt(aerr)
hferr = sqrt(hferr)
q = a
qerr = aerr
return q, qerr, hf, hferr
def RollPrices2Prices(t_end_str, tau, dates, z_roll):
# This function uses rolling values to compute the zero-coupon bond value
# (i.e., discount factors) with maturities in t_end_str.
# INPUTS
# t_end_str [vector]: (k_ x 1) selected maturities (as .Tdd-mmm-yy.T strings)
# tau [vector]: (n_ x 1) times to maturity corresponding to rows of z_roll
# Date [vector]: (1 x t_end) dates corresponding to columns of z_roll
# z_roll [matrix]: (n_ x t_end) rolling values
# OUTPUTS
# date [cell vector]: (k_ x 1) cell date{j} contains the numerical value of
# dates corresponding to columns of z{j}
# z [cell vector]: (k_ x 1) cell z{j} contains the evolution of zero-coupon
# bond value with maturity t_end{j}
# t_end [vector]: (k_ x 1) contains the numerical value corresponding to
# date strings in t_end_string
# tau_int: vector of maturities for interpolation
tauRange = ceil((dates[-1] - dates[0]) / 365)
_, _, tauIndex = intersect(tauRange, tau)
if not tauIndex:
tauIndex = tau.shape[0]
tau_int = arange(tau[0], tau[tauIndex] + tau[0], tau[0])
# declaration and preallocation of variables
t_ = z_roll.shape[1]
n_ = npmax(tau_int.shape)
z_roll_int = zeros((n_, t_))
expiry = zeros((n_, t_))
expiry_f = zeros((n_, t_))
k_ = t_end_str.shape[0]
t_end = zeros((k_, 1), dtype=int)
z = {}
date = {}
for t in range(t_):
# remove zeros
indexPolished = np.where(abs(z_roll[:, t]) > 0)[0]
# matrix of rolling values: z_roll(i,t)=z_{t}(tau[i]+t)
z_roll_int[:, t] = interp(tau_int, tau[indexPolished],
z_roll[indexPolished, t])
# expiries
for i in range(n_):
expiry[i, t] = tau_int[i] * 365 + dates[t]
expiry_f[i, t] = floor(expiry[i, t]) # to remove HH:mm:ss
# zero-coupon bond values (i.e., discount factors) with fixed expiry
for j in range(k_):
z[j] = zeros((1, t_))
date[j] = zeros((1, t_))
t_end[j] = datenum(t_end_str[j])
# z[j] = np.where(expiry_f==t_end[j],z_roll_int,z[j])
# date[j] = np.where(expiry_f==t_end[j],dates,date[j])
for t in range(t_):
for i in range(n_):
if expiry_f[i, t] == t_end[j]:
z[j][0, t] = z_roll_int[i, t]
date[j][0, t] = dates[t]
# remove zeros
indexzeros = np.where(date[j] == 0)
date[j][indexzeros] = np.NAN
z[j][indexzeros] = np.NaN
return date, z, t_end
figure()
# simulated path, mean and standard deviation
plot(horiz_u[:index + 1], PLC_u[:j_sel, :index + 1].T, color=lgrey)
l1 = plot(horiz_u[:index + 1], PLMuC_u[0, :index + 1], color='g')
l2 = plot(horiz_u[:index + 1],
PLMuC_u[0, :index + 1] + PLSigmaC_u[0, :index + 1],
color='r')
plot(horiz_u[:index + 1],
PLMuC_u[0, :index + 1] - PLSigmaC_u[0, :index + 1],
color='r')
# histogram
option = namedtuple('option', 'n_bins')
option.n_bins = round(10 * log(j_))
y_hist, x_hist = HistogramFP(PLC_u[:, [index + 1]].T, pp_, option)
scale = 0.8 * PLSigmaC_u[0, index + 1] / npmax(y_hist)
y_hist = y_hist * scale
shift_y_hist = horiz_u[index + 1] + y_hist
emp_pdf = plt.barh(x_hist[:-1],
shift_y_hist[0] - horiz_u[index + 1],
left=horiz_u[index + 1],
height=x_hist[1] - x_hist[0],
facecolor=lgrey,
edgecolor=lgrey,
lw=2) # empirical pdf
# first order approximation
y_hist1, x_hist1 = HistogramFP(np.atleast_2d(Taylor_first), pp_, option)
y_hist1 = y_hist1 * scale
shift_T_first = horiz_u[index + 1] + y_hist1
def optimal_cmo(hic1,
hic2,
num_v=None,
max_num_v=None,
verbose=False,
method='frobenius',
long_nw=True,
long_dist=True):
"""
Calculates the optimal contact map overlap between 2 matrices
TODO: make the selection of number of eigen vectors automatic or relying on
the summed contribution (e.g. select the EVs that sum 80% of the info)
.. note::
penalty is defined as the minimum value of the pre-scoring matrix.
:param hic1: first matrix to align
:param hic2: second matrix to align
:param None num_v: number of eigen vectors to consider, max is:
max(min(len(hic1), len(hic2)))
:param None max_num_v: maximum number of eigen vectors to consider.
:param score method: distance function to use as alignment score. if 'score'
distance will be the result of the last value of the Needleman-Wunsch
algorithm. If 'frobenius' a modification of the Frobenius distance will
be used
:returns: two lists, one per aligned matrix, plus a dict summarizing the
goodness of the alignment with the distance between matrices, their
Spearman correlation Rho value and pvalue.
"""
l_p1 = len(hic1)
l_p2 = len(hic2)
num_v = num_v or min(l_p1, l_p2)
if max_num_v:
num_v = min(max_num_v, num_v)
if num_v > l_p1 or num_v > l_p2:
raise Exception('\nnum_v should be at most %s\n' % (min(l_p1, l_p2)))
val1, vec1 = eigh(hic1)
if npsum(vec1).imag:
raise Exception("ERROR: Hi-C data is not symmetric.\n" + '%s\n\n%s' %
(hic1, vec1))
val2, vec2 = eigh(hic2)
if npsum(vec2).imag:
raise Exception("ERROR: Hi-C data is not symmetric.\n" + '%s\n\n%s' %
(hic2, vec2))
#
val1 = array([sqrt(abs(v)) for v in val1])
val2 = array([sqrt(abs(v)) for v in val2])
idx = val1.argsort()[::-1]
val1 = val1[idx]
vec1 = vec1[idx]
idx = val2.argsort()[::-1]
val2 = val2[idx]
vec2 = vec2[idx]
#
vec1 = array([val1[i] * vec1[:, i] for i in range(num_v)]).transpose()
vec2 = array([val2[i] * vec2[:, i] for i in range(num_v)]).transpose()
nearest = float('inf')
nw = core_nw_long if long_nw else core_nw
dister = _get_dist_long if long_dist else _get_dist
best_alis = []
for num in range(1, num_v + 1):
for factors in product([1, -1], repeat=num):
vec1p = factors * vec1[:, :num]
vec2p = vec2[:, :num]
p_scores = _prescoring(vec1p, vec2p, l_p1, l_p2)
penalty = min([npmin(p_scores)] + [-npmax(p_scores)])
align1, align2, dist = nw(p_scores, penalty, l_p1, l_p2)
try:
if method == 'frobenius':
dist = dister(align1, align2, hic1, hic2)
else:
dist *= -1
if dist < nearest:
if not penalty:
for scr in p_scores:
print(' '.join(
['%7s' % (round(y, 2)) for y in scr]))
nearest = dist
best_alis = [align1, align2]
best_pen = penalty
except IndexError as e:
print(e)
try:
align1, align2 = best_alis
except ValueError:
pass
if verbose:
print('\n Alignment (score = %s):' % (nearest))
print('TADS 1: '+'|'.join(['%4s' % (str(int(x)) \
if x!='-' else '-'*3) for x in align1]))
print('TADS 2: '+'|'.join(['%4s' % (str(int(x)) \
if x!='-' else '-'*3) for x in align2]))
rho, pval = _get_score(align1, align2, hic1, hic2)
# print best_pen
if not best_pen:
print('WARNING: penalty NULL!!!\n\n')
return align1, align2, {'dist': nearest, 'rho': rho, 'pval': pval}
kappa = z.kappa
sigma_bar = z.sigma_bar
eta = z.eta
rho = z.rho
sigma_0 = z.sigma_0
sigma_heston = sigma_heston*100 # converts values to percentage
sigma_impl = sigma_impl*100 # converts values to percentage
# -
# ## Generate figures showing the evolution of the parameters and the comparison between the realized and the fitted implied volatility surfaces at some points in time
# +
# axes settings
date_tick = linspace(0,t_ - 1,4, dtype=int)
grid_k = linspace(npmin(kappa),npmax(kappa),3)
grid_sigbar = linspace(npmin(sigma_bar),npmax(sigma_bar),3)
grid_lamda = linspace(npmin(eta),npmax(eta),3)
grid_rho = [-1.1, 0, 1.1]
grid_sigma_0 = linspace(min(sigma_0),max(sigma_0),3)
grid_s = [npmin(s), 0.5*(npmin(s) + npmax(s)), npmax(s)]
# volatility fitting plot
n_fig = 1
# number of figures, representing volatility fitting, to be plotted
if n_fig == 1:
t_fig = [0]
else:
t_fig = linspace(0,t_-1,n_fig)
def integrand(x):
"Integral expression on right-hand side of operator"
return npmax(x, phi_f(q(x,pi))) * (pi*f(x) + (1 - pi)*g(x))
f_hor = np.real(f_hor) # ## Compute the normal approximation of the projected pdf phi_hor = norm.pdf(x_hor, mu * tau, sigma * sqrt(tau)) # center around x(t_end) x_hor = x_hor + x[t_ - 1] # ## Create a figure # + s_ = 2 # number of plotted observation before projecting time # axes settings m = min([npmin(x[t_ - s_:t_]), mu_tau - 4.5 * sigma_tau]) M = max([npmax(x[t_ - s_:t_]), mu_tau + 5 * sigma_tau]) t = arange(-s_, tau) max_scale = tau / 4 # preliminary computations tau_red = arange(0, tau + 0.1, 0.1) mu_red = x[t_ - 1] + mu * tau_red sigma_red = sigma * sqrt(tau_red) redline1 = mu_red + 2 * sigma_red redline2 = mu_red - 2 * sigma_red f = figure() # color settings lgrey = [0.8, 0.8, 0.8] # light grey dgrey = [0.2, 0.2, 0.2] # dark grey
# Generate scenarios for the invariants via historical bootstrapping epsi_hor = SampleScenProbDistribution(epsi.reshape(1,-1), p, j_*deltat) epsi_hor = reshape(epsi_hor, (j_, deltat),'F') # Feed the simulated scenarios in the recursive incremental-step routine((random walk assumption)) pi_deltat = pi[-1] + cumsum(epsi_hor, 1) # - # ## Create a figure # + s_ = 2 # number of plotted observation before projecting time # axes settings m = min([npmin(pi[::-1]), pi[-1]-4*sigma_tau]) M = max([npmax(pi[::-1]), mu_tau + 4.5*sigma_tau]) t = arange(-s_,deltat+1) max_scale = deltat / 4 # preliminary computations tau_red = arange(0,deltat+0.1,0.1) mu_red = pi[-1] + mu_1*tau_red sigma_red = sigma_1*sqrt(tau_red) redline1 = mu_red + 2*sigma_red redline2 = mu_red - 2*sigma_red from matplotlib.pyplot import xticks f = figure() # color settings lgrey = [0.8, 0.8, 0.8] # light grey
# + phi_tau = norm.pdf(epsi_tau, mu * tau, sigma * sqrt(tau)) # center around x[t_end-1] epsi_tau = epsi_tau + x[t_ - 1] # - # ## Create a figure # + s_ = 2 # number of plotted observation before projecting time # axes settings m = min([npmin(x[t_ - 2:t_]), x[t_] - 4 * sigma_tau]) M = max([npmax(x[t_ - 2:t_]), x[t_] + mu_tau + 4.5 * sigma_tau]) t = arange(-s_, tau + 1) max_scale = tau / 4 scale = max_scale / npmax(f_tau) # preliminary computations tau_red = arange(0, tau + 0.1, 0.1) mu_red = x[t_ - 1] + mu * tau_red sigma_red = sigma * sqrt(tau_red) redline1 = mu_red + 2 * sigma_red redline2 = mu_red - 2 * sigma_red f = figure() # color settings lgrey = [0.8, 0.8, 0.8] # light grey dgrey = [0.2, 0.2, 0.2] # dark grey
xlabel('$\epsilon_1$')
ylabel('$\epsilon_2$')
PlotTwoDimEllipsoid(mu_HFP[:, [q]], sigma2_HFP[:, :, q], 1, 0, 0, 'r', 2)
# histogram of z^2
options = namedtuple('option', 'n_bins')
options.n_bins = round(30 * log(ens[0, q]))
plt.sca(ax[0, 1])
ax[0, 1].set_facecolor('white')
nz, zz = HistogramFP(z_2[[q], :], P.reshape(1, -1), options)
b = bar(zz[:-1],
nz[0],
width=zz[1] - zz[0],
facecolor=[.7, .7, .7],
edgecolor=[.3, .3, .3])
plt.axis([-1, 15, 0, npmax(nz) + (npmax(nz) / 20)])
yticks([])
xlabel('$z^2$')
plot(mu_z2[0, q],
0,
color='r',
marker='o',
markerfacecolor='r',
markersize=4)
MZ2 = 'HFP - mean($z^2$) = % 3.2f' % mu_z2[0, q]
plt.text(15,
npmax(nz) - (npmax(nz) / 7),
MZ2,
color='r',
horizontalalignment='right',
Rates.idx, tau + 1, :] # shadow yield curve at the projection step tau
interp = interp1d(Rates.tau, ShadowRates_thor.T, fill_value='extrapolate')
Shadowy_thor = interp(
Bonds.tau_thor
).T # interpolate the curve to obtain the shadow rates for the relevant time to maturity
Y_thor = PerpetualAmericanCall(
Shadowy_thor, {'eta': Rates.eta}) # from shadow yields to yields
Bonds.V_thor = zeros((Bonds.n_, Y_thor.shape[1]))
Bonds.V_mc_thor = zeros((Bonds.n_, Y_thor.shape[1]))
for n in range(Bonds.n_):
Bonds.V_thor[n, :] = exp(-Bonds.tau_thor[n] * Y_thor[n, :])
Bonds.V_mc_thor[n, :] = npmax(
Bonds.I_D[n, :, :] * Bonds.recoveryrates[n] * Bonds.EAD[n, :, :],
1).T + (1 - Bonds.I_D[n, :, -1]) * Bonds.V_thor[n, :]
# P&L's
Bonds.Pi = Bonds.V_mc_thor - tile(Bonds.v_tnow, (1, j_))
# -
# ## Pricing: Call options
# +
Options.strikes = array([1100, 1150, 1200])
# Implied volatility paths (reshaped)
implvol_idx = arange(Stocks.i_ + Bonds.i_ + 1, i_)
LogImplVol_path = reshape(
def gen_plot(self, args):
"""Generate the plot from time series and arguments
"""
self.max_size = 16384. * 6400. # that works on my mac
self.yscale_factor = 1.0
from gwpy.plotter.tex import label_to_latex
from numpy import min as npmin
from numpy import max as npmax
if self.timeseries[0].size <= self.max_size:
self.plot = self.timeseries[0].plot()
else:
self.plot = self.timeseries[0].plot(linestyle='None', marker='.')
self.ymin = self.timeseries[0].min().value
self.ymax = self.timeseries[0].max().value
self.xmin = self.timeseries[0].times.value.min()
self.xmax = self.timeseries[0].times.value.max()
if len(self.timeseries) > 1:
for idx in range(1, len(self.timeseries)):
chname = self.timeseries[idx].channel.name
lbl = label_to_latex(chname)
if self.timeseries[idx].size <= self.max_size:
self.plot.add_timeseries(self.timeseries[idx], label=lbl)
else:
self.plot.add_timeseries(self.timeseries[idx], label=lbl,
linestyle='None', marker='.')
self.ymin = min(self.ymin, self.timeseries[idx].min().value)
self.ymax = max(self.ymax, self.timeseries[idx].max().value)
self.xmin = min(self.xmin,
self.timeseries[idx].times.value.min())
self.xmax = max(self.xmax,
self.timeseries[idx].times.value.max())
# if they chose to set the range of the x-axis find the range of y
strt = self.xmin
stop = self.xmax
# a bit weird but global ymax will be >= any value in
# the range same for ymin
new_ymin = self.ymax
new_ymax = self.ymin
if args.xmin:
strt = float(args.xmin)
if args.xmax:
stop = float(args.xmax)
if strt != self.xmin or stop != self.xmax:
for idx in range(0, len(self.timeseries)):
x0 = self.timeseries[idx].x0.value
dt = self.timeseries[idx].dt.value
if strt < 1e8:
strt += x0
if stop < 1e8:
stop += x0
b = int(max(0, (strt - x0) / dt))
e = int(min(self.xmax, (stop - x0) / dt))
if e >= self.timeseries[idx].size:
e = self.timeseries[idx].size - 1
new_ymin = min(new_ymin,
npmin(self.timeseries[idx].value[b:e]))
new_ymax = max(new_ymax,
npmax(self.timeseries[idx].value[b:e]))
self.ymin = new_ymin
self.ymax = new_ymax
if self.yscale_factor > 1:
self.log(2, ('Scaling y-limits, original: %f, %f)' %
(self.ymin, self.ymax)))
yrange = self.ymax - self.ymin
mid = (self.ymax + self.ymin) / 2.
self.ymax = mid + yrange / (2 * self.yscale_factor)
self.ymin = mid - yrange / (2 * self.yscale_factor)
self.log(2, ('Scaling y-limits, new: %f, %f)' %
(self.ymin, self.ymax)))
def PlotDynamicStrats(t, V_t_strat, V_t_risky, W_t_risky):
## This function generates two figures. The first plots the evolution of a
# dynamic strategy of two instruments and the weights on the risky asset.
# The second one is the scatter/histogram plot of the payoffs of the
# strategy and the risky asset.
#
# INPUTS
# t :[vector] (n_bar x 1) vector of time
# V_t :[matrix] (n_bar x j_bar) portfolio scenarios
# V_t_risky :[matrix] (n_bar x j_bar) risky instrument scenarios
# W_t_risky :[matrix] (n_bar x j_bar) weight of the risky instrument
# For details on the exercise, see here .
## Code
# adjust V_t_risky so that it has the same initial value as the strategy
V_t_risky = V_t_risky * V_t_strat[0, 0] / V_t_risky[0, 0]
## Plot the values and the weights
j = 1 # select one scenario
y_max = npmax([V_t_strat[:, j], V_t_risky[:, j]
]) * 1.2 # maximum of the y-axis
fig1 = figure()
# plot the scenario
plt.subplot(2, 1, 1)
plot(t, V_t_strat[:, j], lw=2.5, color='b')
plot(t, V_t_risky[:, j], lw=2, color='r')
plt.axis([0, t[-1], 0, y_max])
plt.grid(True)
ylabel('value')
title('investment (blue) vs underlying (red) value')
# bar plot of the weights
plt.subplot(2, 1, 2)
bar(t, W_t_risky[:, j], width=t[1] - t[0], color='r', edgecolor='k')
plt.axis([0, t[-1], 0, 1])
plt.grid(True)
xlabel('time')
ylabel('$')
title('percentage of underlying in portfolio')
plt.tight_layout()
## Joint scatter/histogram plot for the payoffs
fig2 = figure()
NumBins = int(round(10 * log(V_t_strat.shape[1])))
ax = plt.subplot2grid((4, 4), (0, 0), rowspan=3)
# histograms
[n, D] = histogram(V_t_strat[-1, :], NumBins)
barh(D[:-1], n, height=D[1] - D[0])
xticks([])
plt.grid(True)
y_lim = plt.ylim()
ax = plt.subplot2grid((4, 4), (3, 1), colspan=3)
[n, D] = histogram(V_t_risky[-1, :], NumBins)
bar(D[:-1], n, width=D[1] - D[0])
yticks([])
plt.grid(True)
x_lim = plt.xlim()
# scatter plot
ax = plt.subplot2grid((4, 4), (0, 1), rowspan=3, colspan=3)
scatter(V_t_risky[-1, :], V_t_strat[-1, :], marker='.', s=2)
so = sort(V_t_risky[-1, :])
plot(so, so, 'r')
xlim(x_lim)
ylim(y_lim)
plt.grid(True)
xlabel('underlying at horizon')
ylabel('investment at horizon')
plt.tight_layout()
return fig1, fig2
empty_flag="True"
print 'x: ', x
print 'y: ', y
else:
coverage_lines=stdout.split('\n')[:-1]
x,y=unfold_bedgraph(coverage_lines)
meancov=mean(y)
#### gaussian on the original peak interval #####
fitgaus=fit_gaussian(y,x)
a,b,c,best,fwhm,chisq_raw,reduced_chisq=fitgaus
area=area_under_curve(best)
width95CI,fit_height,ht_to_wid_ratio=get_peak_stats(fitgaus,area)
plateau_calc=plateauiness(best)
normalized_y=1000*array(y)/float(npmax(y))
norm_fitgaus=fit_gaussian(normalized_y,x)
norm_a,norm_b,_norm_c,norm_best,norm_fwhm,norm_chisq_raw,norm_reduced_chisq=norm_fitgaus
norm_area=area_under_curve(best)
norm_width95CI,norm_fit_height,norm_ht_to_wid_ratio=get_peak_stats(norm_fitgaus,norm_area)
#########
### evaluate possibility of double peaks
double_check=eval_double_peak(x,y)
if len(double_check)==2:
bic1,bic2=double_check
else:
x_gaus1,y_gaus1,x_gaus2,y_gaus2,bic1,bic2=double_check
meancov_1=mean(y_gaus1)
meancov_2=mean(y_gaus2)
### 1st gaussian in mixture ###
def optimal_cmo(hic1, hic2, num_v=None, max_num_v=None, verbose=False,
method='frobenius', long_nw=True, long_dist=True):
"""
Calculates the optimal contact map overlap between 2 matrices
TODO: make the selection of number of eigen vectors automatic or relying on
the summed contribution (e.g. select the EVs that sum 80% of the info)
.. note::
penalty is defined as the minimum value of the pre-scoring matrix.
:param hic1: first matrix to align
:param hic2: second matrix to align
:param None num_v: number of eigen vectors to consider, max is:
max(min(len(hic1), len(hic2)))
:param None max_num_v: maximum number of eigen vectors to consider.
:param score method: distance function to use as alignment score. if 'score'
distance will be the result of the last value of the Needleman-Wunsch
algorithm. If 'frobenius' a modification of the Frobenius distance will
be used
:returns: two lists, one per aligned matrix, plus a dict summarizing the
goodness of the alignment with the distance between matrices, their
Spearman correlation Rho value and pvalue.
"""
l_p1 = len(hic1)
l_p2 = len(hic2)
num_v = num_v or min(l_p1, l_p2)
if max_num_v:
num_v = min(max_num_v, num_v)
if num_v > l_p1 or num_v > l_p2:
raise Exception('\nnum_v should be at most %s\n' % (min(l_p1, l_p2)))
val1, vec1 = eigh(hic1)
if npsum(vec1).imag:
raise Exception("ERROR: Hi-C data is not symmetric.\n" +
'%s\n\n%s' % (hic1, vec1))
val2, vec2 = eigh(hic2)
if npsum(vec2).imag:
raise Exception("ERROR: Hi-C data is not symmetric.\n" +
'%s\n\n%s' % (hic2, vec2))
#
val1 = array([sqrt(abs(v)) for v in val1])
val2 = array([sqrt(abs(v)) for v in val2])
idx = val1.argsort()[::-1]
val1 = val1[idx]
vec1 = vec1[idx]
idx = val2.argsort()[::-1]
val2 = val2[idx]
vec2 = vec2[idx]
#
vec1 = array([val1[i] * vec1[:, i] for i in xrange(num_v)]).transpose()
vec2 = array([val2[i] * vec2[:, i] for i in xrange(num_v)]).transpose()
nearest = float('inf')
nw = core_nw_long if long_nw else core_nw
dister = _get_dist_long if long_dist else _get_dist
best_alis = []
for num in xrange(1, num_v + 1):
for factors in product([1, -1], repeat=num):
vec1p = factors * vec1[:, :num]
vec2p = vec2[:, :num]
p_scores = _prescoring(vec1p, vec2p, l_p1, l_p2)
penalty = min([npmin(p_scores)] + [-npmax(p_scores)])
align1, align2, dist = nw(p_scores, penalty, l_p1, l_p2)
try:
if method == 'frobenius':
dist = dister(align1, align2, hic1, hic2)
else:
dist *= -1
if dist < nearest:
if not penalty:
for scr in p_scores:
print ' '.join(['%7s' % (round(y, 2)) for y in scr])
nearest = dist
best_alis = [align1, align2]
best_pen = penalty
except IndexError as e:
print e
try:
align1, align2 = best_alis
except ValueError:
pass
if verbose:
print '\n Alignment (score = %s):' % (nearest)
print 'TADS 1: '+'|'.join(['%4s' % (str(int(x)) \
if x!='-' else '-'*3) for x in align1])
print 'TADS 2: '+'|'.join(['%4s' % (str(int(x)) \
if x!='-' else '-'*3) for x in align2])
rho, pval = _get_score(align1, align2, hic1, hic2)
# print best_pen
if not best_pen:
print 'WARNING: penalty NULL!!!\n\n'
return align1, align2, {'dist': nearest, 'rho': rho, 'pval': pval}
def lightcurve_flux_measures(ftimes, fmags, ferrs, magsarefluxes=False):
'''
This calculates percentiles of the flux.
'''
ndet = len(fmags)
if ndet > 9:
# get the fluxes
if magsarefluxes:
series_fluxes = fmags
else:
series_fluxes = 10.0**(-0.4 * fmags)
series_flux_median = npmedian(series_fluxes)
# get the percent_amplitude for the fluxes
series_flux_percent_amplitude = (npmax(npabs(series_fluxes)) /
series_flux_median)
# get the flux percentiles
series_flux_percentiles = nppercentile(
series_fluxes,
[5.0, 10, 17.5, 25, 32.5, 40, 60, 67.5, 75, 82.5, 90, 95])
series_frat_595 = (series_flux_percentiles[-1] -
series_flux_percentiles[0])
series_frat_1090 = (series_flux_percentiles[-2] -
series_flux_percentiles[1])
series_frat_175825 = (series_flux_percentiles[-3] -
series_flux_percentiles[2])
series_frat_2575 = (series_flux_percentiles[-4] -
series_flux_percentiles[3])
series_frat_325675 = (series_flux_percentiles[-5] -
series_flux_percentiles[4])
series_frat_4060 = (series_flux_percentiles[-6] -
series_flux_percentiles[5])
# calculate the flux percentile ratios
series_flux_percentile_ratio_mid20 = series_frat_4060 / series_frat_595
series_flux_percentile_ratio_mid35 = series_frat_325675 / series_frat_595
series_flux_percentile_ratio_mid50 = series_frat_2575 / series_frat_595
series_flux_percentile_ratio_mid65 = series_frat_175825 / series_frat_595
series_flux_percentile_ratio_mid80 = series_frat_1090 / series_frat_595
# calculate the ratio of F595/median flux
series_percent_difference_flux_percentile = (series_frat_595 /
series_flux_median)
series_percentile_magdiff = -2.5 * nplog10(
series_percent_difference_flux_percentile)
return {
'flux_median': series_flux_median,
'flux_percent_amplitude': series_flux_percent_amplitude,
'flux_percentiles': series_flux_percentiles,
'flux_percentile_ratio_mid20': series_flux_percentile_ratio_mid20,
'flux_percentile_ratio_mid35': series_flux_percentile_ratio_mid35,
'flux_percentile_ratio_mid50': series_flux_percentile_ratio_mid50,
'flux_percentile_ratio_mid65': series_flux_percentile_ratio_mid65,
'flux_percentile_ratio_mid80': series_flux_percentile_ratio_mid80,
'percent_difference_flux_percentile': series_percentile_magdiff,
}
else:
LOGERROR('not enough detections in this magseries '
'to calculate flux measures')
return None
def nonperiodic_lightcurve_features(times, mags, errs, magsarefluxes=False):
'''This calculates the following nonperiodic features of the light curve,
listed in Richards, et al. 2011):
amplitude
beyond1std
flux_percentile_ratio_mid20
flux_percentile_ratio_mid35
flux_percentile_ratio_mid50
flux_percentile_ratio_mid65
flux_percentile_ratio_mid80
linear_trend
max_slope
median_absolute_deviation
median_buffer_range_percentage
pair_slope_trend
percent_amplitude
percent_difference_flux_percentile
skew
stdev
timelength
mintime
maxtime
'''
# remove nans first
finiteind = npisfinite(times) & npisfinite(mags) & npisfinite(errs)
ftimes, fmags, ferrs = times[finiteind], mags[finiteind], errs[finiteind]
# remove zero errors
nzind = npnonzero(ferrs)
ftimes, fmags, ferrs = ftimes[nzind], fmags[nzind], ferrs[nzind]
ndet = len(fmags)
if ndet > 9:
# calculate the moments
moments = lightcurve_moments(ftimes, fmags, ferrs)
# calculate the flux measures
fluxmeasures = lightcurve_flux_measures(ftimes,
fmags,
ferrs,
magsarefluxes=magsarefluxes)
# calculate the point-to-point measures
ptpmeasures = lightcurve_ptp_measures(ftimes, fmags, ferrs)
# get the length in time
mintime, maxtime = npmin(ftimes), npmax(ftimes)
timelength = maxtime - mintime
# get the amplitude
series_amplitude = 0.5 * (npmax(fmags) - npmin(fmags))
# calculate the linear fit to the entire mag series
fitcoeffs = nppolyfit(ftimes, fmags, 1, w=1.0 / (ferrs * ferrs))
series_linear_slope = fitcoeffs[1]
# roll fmags by 1
rolled_fmags = nproll(fmags, 1)
# calculate the magnitude ratio (from the WISE paper)
series_magratio = ((npmax(fmags) - moments['median']) /
(npmax(fmags) - npmin(fmags)))
# this is the dictionary returned containing all the measures
measures = {
'ndet': fmags.size,
'mintime': mintime,
'maxtime': maxtime,
'timelength': timelength,
'amplitude': series_amplitude,
'ndetobslength_ratio': ndet / timelength,
'linear_fit_slope': series_linear_slope,
'magnitude_ratio': series_magratio,
}
if moments:
measures.update(moments)
if ptpmeasures:
measures.update(ptpmeasures)
if fluxmeasures:
measures.update(fluxmeasures)
return measures
else:
LOGERROR('not enough detections in this magseries '
'to calculate non-periodic features')
return None
def find_compartments(self,
crms=None,
savefig=None,
savedata=None,
show=False,
**kwargs):
"""
Search for A/B copartments in each chromsome of the Hi-C matrix.
Hi-C matrix is normalized by the number interaction expected at a given
distance, and by visibility (one iteration of ICE). A correlation matrix
is then calculated from this normalized matrix, and its first
eigenvector is used to identify compartments. Changes in sign marking
boundaries between compartments.
Result is stored as a dictionary of compartment boundaries, keys being
chromsome names.
:param 99 perc_zero: to filter bad columns
:param 0.05 signal_to_noise: to calculate expected interaction counts,
if not enough reads are observed at a given distance the observations
of the distance+1 are summed. a signal to noise ratio of < 0.05
corresponds to > 400 reads.
:param None crms: only runs these given list of chromosomes
:param None savefig: path to a directory to store matrices with
compartment predictions, one image per chromosome, stored under
'chromosome-name.png'.
:param False show: show the plot
:param None savedata: path to a new file to store compartment
predictions, one file only.
:param -1 vmin: for the color scale of the plotted map
:param 1 vmax: for the color scale of the plotted map
TODO: this is really slow...
Notes: building the distance matrix using the amount of interactions
instead of the mean correlation, gives generally worse results.
"""
if not self.bads:
if kwargs.get('verbose', True):
print 'Filtering bad columns %d' % 99
self.filter_columns(perc_zero=kwargs.get('perc_zero', 99),
by_mean=False,
silent=True)
if not self.expected:
if kwargs.get('verbose', True):
print 'Normalizing by expected values'
self.expected = expected(self, bads=self.bads, **kwargs)
if not self.bias:
if kwargs.get('verbose', True):
print 'Normalizing by ICE (1 round)'
self.normalize_hic(iterations=0)
if savefig:
mkdir(savefig)
cmprts = {}
for sec in self.section_pos:
if crms and sec not in crms:
continue
if kwargs.get('verbose', False):
print 'Processing chromosome', sec
warn('Processing chromosome %s' % (sec))
matrix = [[(float(self[i, j]) / self.expected[abs(j - i)] /
self.bias[i] / self.bias[j])
for i in xrange(*self.section_pos[sec])
if not i in self.bads]
for j in xrange(*self.section_pos[sec])
if not j in self.bads]
if not matrix: # MT chromosome will fall there
warn('Chromosome %s is probably MT :)' % (sec))
cmprts[sec] = []
continue
for i in xrange(len(matrix)):
for j in xrange(i + 1, len(matrix)):
matrix[i][j] = matrix[j][i]
matrix = [list(m) for m in corrcoef(matrix)]
try:
# This eighs is very very fast, only ask for one eigvector
_, evect = eigsh(array(matrix), k=1)
except LinAlgError:
warn('Chromosome %s too small to compute PC1' % (sec))
cmprts[sec] = [] # Y chromosome, or so...
continue
first = list(evect[:, -1])
beg, end = self.section_pos[sec]
bads = [k - beg for k in self.bads if beg <= k <= end]
_ = [first.insert(b, 0) for b in bads]
_ = [
matrix.insert(b, [float('nan')] * len(matrix[0])) for b in bads
]
_ = [
matrix[i].insert(b, float('nan')) for b in bads
for i in xrange(len(first))
]
breaks = [0] + [
i for i, (a, b) in enumerate(zip(first[1:], first[:-1]))
if a * b < 0
] + [len(first)]
breaks = [{
'start': b,
'end': breaks[i + 1]
} for i, b in enumerate(breaks[:-1])]
cmprts[sec] = breaks
# calculate compartment internal density
for k, cmprt in enumerate(cmprts[sec]):
beg = self.section_pos[sec][0]
beg1, end1 = cmprt['start'] + beg, cmprt['end'] + beg
sec_matrix = [(self[i, j] / self.expected[abs(j - i)] /
self.bias[i] / self.bias[j])
for i in xrange(beg1, end1) if not i in self.bads
for j in xrange(i, end1) if not j in self.bads]
try:
cmprt['dens'] = sum(sec_matrix) / len(sec_matrix)
except ZeroDivisionError:
cmprt['dens'] = 0.
try:
meanh = sum([cmprt['dens']
for cmprt in cmprts[sec]]) / len(cmprts[sec])
except ZeroDivisionError:
meanh = 1.
for cmprt in cmprts[sec]:
try:
cmprt['dens'] /= meanh
except ZeroDivisionError:
cmprt['dens'] = 1.
gammas = {}
for gamma in range(101):
gammas[gamma] = _find_ab_compartments(float(gamma) / 100,
matrix,
breaks,
cmprts[sec],
save=False)
# print gamma, gammas[gamma]
gamma = min(gammas.keys(), key=lambda k: gammas[k][0])
_ = _find_ab_compartments(float(gamma) / 100,
matrix,
breaks,
cmprts[sec],
save=True)
if savefig or show:
vmin = kwargs.get('vmin', -1)
vmax = kwargs.get('vmax', 1)
if vmin == 'auto' == vmax:
vmax = max([abs(npmin(matrix)), abs(npmax(matrix))])
vmin = -vmax
plot_compartments(sec,
first,
cmprts,
matrix,
show,
savefig + '/chr' + sec + '.pdf',
vmin=vmin,
vmax=vmax)
plot_compartments_summary(sec, cmprts, show,
savefig + '/chr' + sec + '_summ.pdf')
self.compartments = cmprts
if savedata:
self.write_compartments(savedata)
def plot_phased_mag_series(times,
mags,
period,
errs=None,
epoch='min',
outfile=None,
sigclip=30.0,
phasewrap=True,
phasesort=True,
phasebin=None,
plotphaselim=[-0.8,0.8],
yrange=None):
'''This plots a phased magnitude time series using the period provided.
If epoch is None, uses the min(times) as the epoch.
If epoch is a string 'min', then fits a cubic spline to the phased light
curve using min(times), finds the magnitude minimum from the fitted light
curve, then uses the corresponding time value as the epoch.
If epoch is a float, then uses that directly to phase the light curve and as
the epoch of the phased mag series plot.
If outfile is none, then plots to matplotlib interactive window. If outfile
is a string denoting a filename, uses that to write a png/eps/pdf figure.
'''
if errs is not None:
# remove nans
find = npisfinite(times) & npisfinite(mags) & npisfinite(errs)
ftimes, fmags, ferrs = times[find], mags[find], errs[find]
# get the median and stdev = 1.483 x MAD
median_mag = npmedian(fmags)
stddev_mag = (npmedian(npabs(fmags - median_mag))) * 1.483
# sigclip next
if sigclip:
sigind = (npabs(fmags - median_mag)) < (sigclip * stddev_mag)
stimes = ftimes[sigind]
smags = fmags[sigind]
serrs = ferrs[sigind]
LOGINFO('sigclip = %s: before = %s observations, '
'after = %s observations' %
(sigclip, len(times), len(stimes)))
else:
stimes = ftimes
smags = fmags
serrs = ferrs
else:
# remove nans
find = npisfinite(times) & npisfinite(mags)
ftimes, fmags, ferrs = times[find], mags[find], None
# get the median and stdev = 1.483 x MAD
median_mag = npmedian(fmags)
stddev_mag = (npmedian(npabs(fmags - median_mag))) * 1.483
# sigclip next
if sigclip:
sigind = (npabs(fmags - median_mag)) < (sigclip * stddev_mag)
stimes = ftimes[sigind]
smags = fmags[sigind]
serrs = None
LOGINFO('sigclip = %s: before = %s observations, '
'after = %s observations' %
(sigclip, len(times), len(stimes)))
else:
stimes = ftimes
smags = fmags
serrs = None
# figure out the epoch, if it's None, use the min of the time
if epoch is None:
epoch = npmin(stimes)
# if the epoch is 'min', then fit a spline to the light curve phased
# using the min of the time, find the fit mag minimum and use the time for
# that as the epoch
elif isinstance(epoch,str) and epoch == 'min':
spfit = spline_fit_magseries(stimes, smags, serrs, period)
epoch = spfit['fitepoch']
# now phase (and optionally, phase bin the light curve)
if errs is not None:
# phase the magseries
phasedlc = phase_magseries_with_errs(stimes,
smags,
serrs,
period,
epoch,
wrap=phasewrap,
sort=phasesort)
plotphase = phasedlc['phase']
plotmags = phasedlc['mags']
ploterrs = phasedlc['errs']
# if we're supposed to bin the phases, do so
if phasebin:
binphasedlc = phase_bin_magseries_with_errs(plotphase,
plotmags,
ploterrs,
binsize=phasebin)
plotphase = binphasedlc['binnedphases']
plotmags = binphasedlc['binnedmags']
ploterrs = binphasedlc['binnederrs']
else:
# phase the magseries
phasedlc = phase_magseries(stimes,
smags,
period,
epoch,
wrap=phasewrap,
sort=phasesort)
plotphase = phasedlc['phase']
plotmags = phasedlc['mags']
ploterrs = None
# if we're supposed to bin the phases, do so
if phasebin:
binphasedlc = phase_bin_magseries(plotphase,
plotmags,
binsize=phasebin)
plotphase = binphasedlc['binnedphases']
plotmags = binphasedlc['binnedmags']
ploterrs = None
# finally, make the plots
# initialize the plot
fig = plt.figure()
fig.set_size_inches(7.5,4.8)
plt.errorbar(plotphase, plotmags, fmt='bo', yerr=ploterrs,
markersize=2.0, markeredgewidth=0.0, ecolor='#B2BEB5',
capsize=0)
# make a grid
plt.grid(color='#a9a9a9',
alpha=0.9,
zorder=0,
linewidth=1.0,
linestyle=':')
# make lines for phase 0.0, 0.5, and -0.5
plt.axvline(0.0,alpha=0.9,linestyle='dashed',color='g')
plt.axvline(-0.5,alpha=0.9,linestyle='dashed',color='g')
plt.axvline(0.5,alpha=0.9,linestyle='dashed',color='g')
# fix the ticks to use no offsets
plt.gca().get_yaxis().get_major_formatter().set_useOffset(False)
plt.gca().get_xaxis().get_major_formatter().set_useOffset(False)
# get the yrange
if yrange and isinstance(yrange,list) and len(yrange) == 2:
ymin, ymax = yrange
else:
ymin, ymax = plt.ylim()
plt.ylim(ymax,ymin)
# set the x axis limit
if not plotphaselim:
plot_xlim = plt.xlim()
plt.xlim((npmin(plotphase)-0.1,
npmax(plotphase)+0.1))
else:
plt.xlim((plotphaselim[0],plotphaselim[1]))
# set up the labels
plt.xlabel('phase')
plt.ylabel('magnitude')
plt.title('using period: %.6f d and epoch: %.6f' % (period, epoch))
# make the figure
if outfile and isinstance(outfile, str):
plt.savefig(outfile,bbox_inches='tight')
plt.close()
return os.path.abspath(outfile)
else:
plt.show()
plt.close()
return
def floor_sig(sig, sig_min=1e-8):
r"""Floor an array of variances
"""
sig_floor = npmin([norm(sig), sig_min])
return list(map(lambda s: npmax([s, sig_floor]), sig))
def find_compartments(self, crms=None, savefig=None, savedata=None,
show=False, **kwargs):
"""
Search for A/B copartments in each chromsome of the Hi-C matrix.
Hi-C matrix is normalized by the number interaction expected at a given
distance, and by visibility (one iteration of ICE). A correlation matrix
is then calculated from this normalized matrix, and its first
eigenvector is used to identify compartments. Changes in sign marking
boundaries between compartments.
Result is stored as a dictionary of compartment boundaries, keys being
chromsome names.
:param 99 perc_zero: to filter bad columns
:param 0.05 signal_to_noise: to calculate expected interaction counts,
if not enough reads are observed at a given distance the observations
of the distance+1 are summed. a signal to noise ratio of < 0.05
corresponds to > 400 reads.
:param None crms: only runs these given list of chromosomes
:param None savefig: path to a directory to store matrices with
compartment predictions, one image per chromosome, stored under
'chromosome-name.png'.
:param False show: show the plot
:param None savedata: path to a new file to store compartment
predictions, one file only.
:param -1 vmin: for the color scale of the plotted map
:param 1 vmax: for the color scale of the plotted map
TODO: this is really slow...
Notes: building the distance matrix using the amount of interactions
instead of the mean correlation, gives generally worse results.
"""
if not self.bads:
if kwargs.get('verbose', True):
print 'Filtering bad columns %d' % 99
self.filter_columns(perc_zero=kwargs.get('perc_zero', 99),
by_mean=False, silent=True)
if not self.expected:
if kwargs.get('verbose', True):
print 'Normalizing by expected values'
self.expected = expected(self, bads=self.bads, **kwargs)
if not self.bias:
if kwargs.get('verbose', True):
print 'Normalizing by ICE (1 round)'
self.normalize_hic(iterations=0)
if savefig:
mkdir(savefig)
cmprts = {}
for sec in self.section_pos:
if crms and sec not in crms:
continue
if kwargs.get('verbose', False):
print 'Processing chromosome', sec
warn('Processing chromosome %s' % (sec))
matrix = [[(float(self[i,j]) / self.expected[abs(j-i)]
/ self.bias[i] / self.bias[j])
for i in xrange(*self.section_pos[sec])
if not i in self.bads]
for j in xrange(*self.section_pos[sec])
if not j in self.bads]
if not matrix: # MT chromosome will fall there
warn('Chromosome %s is probably MT :)' % (sec))
cmprts[sec] = []
continue
for i in xrange(len(matrix)):
for j in xrange(i+1, len(matrix)):
matrix[i][j] = matrix[j][i]
matrix = [list(m) for m in corrcoef(matrix)]
try:
# This eighs is very very fast, only ask for one eigvector
_, evect = eigsh(array(matrix), k=1)
except LinAlgError:
warn('Chromosome %s too small to compute PC1' % (sec))
cmprts[sec] = [] # Y chromosome, or so...
continue
first = list(evect[:, -1])
beg, end = self.section_pos[sec]
bads = [k - beg for k in self.bads if beg <= k <= end]
_ = [first.insert(b, 0) for b in bads]
_ = [matrix.insert(b, [float('nan')] * len(matrix[0]))
for b in bads]
_ = [matrix[i].insert(b, float('nan'))
for b in bads for i in xrange(len(first))]
breaks = [0] + [i for i, (a, b) in
enumerate(zip(first[1:], first[:-1]))
if a * b < 0] + [len(first)]
breaks = [{'start': b, 'end': breaks[i+1]}
for i, b in enumerate(breaks[: -1])]
cmprts[sec] = breaks
# calculate compartment internal density
for k, cmprt in enumerate(cmprts[sec]):
beg = self.section_pos[sec][0]
beg1, end1 = cmprt['start'] + beg, cmprt['end'] + beg
sec_matrix = [(self[i,j] / self.expected[abs(j-i)]
/ self.bias[i] / self.bias[j])
for i in xrange(beg1, end1) if not i in self.bads
for j in xrange(i, end1) if not j in self.bads]
try:
cmprt['dens'] = sum(sec_matrix) / len(sec_matrix)
except ZeroDivisionError:
cmprt['dens'] = 0.
try:
meanh = sum([cmprt['dens'] for cmprt in cmprts[sec]]) / len(cmprts[sec])
except ZeroDivisionError:
meanh = 1.
for cmprt in cmprts[sec]:
try:
cmprt['dens'] /= meanh
except ZeroDivisionError:
cmprt['dens'] = 1.
gammas = {}
for gamma in range(101):
gammas[gamma] = _find_ab_compartments(float(gamma)/100, matrix,
breaks, cmprts[sec],
save=False)
# print gamma, gammas[gamma]
gamma = min(gammas.keys(), key=lambda k: gammas[k][0])
_ = _find_ab_compartments(float(gamma)/100, matrix, breaks,
cmprts[sec], save=True)
if savefig or show:
vmin = kwargs.get('vmin', -1)
vmax = kwargs.get('vmax', 1)
if vmin == 'auto' == vmax:
vmax = max([abs(npmin(matrix)), abs(npmax(matrix))])
vmin = -vmax
plot_compartments(sec, first, cmprts, matrix, show,
savefig + '/chr' + sec + '.pdf',
vmin=vmin, vmax=vmax)
plot_compartments_summary(sec, cmprts, show,
savefig + '/chr' + sec + '_summ.pdf')
self.compartments = cmprts
if savedata:
self.write_compartments(savedata)
def __remove_small_peaks(data, intervals, lmax, factor=10):
small = (lmax < (npmax(data) / factor))
for is_small, (min, max) in izip(small, intervals):
if is_small:
data[min:max] = 0
return data
def plateauiness(y_vector,prop_max=0.80):
plat_stat=sum(i>=npmax(y_vector)*prop_max for i in y_vector)
return plat_stat
# input parameters t_ = 500 # number of observations nu = 3 # degrees of freedom mu = 0 # location parameter sigma2 = 2 # square dispersion parameter sigma = sqrt(sigma2) threshold = 1e-4 last = 1 # - # ## Generate the observation of the Student t with 3 degree of freedom, location parameter 0 and dispersion parameter 2 Epsi_std = t.rvs(nu, size=(1, t_)) Epsi = mu + sigma * Epsi_std # Affine equivariance property x = linspace(npmin(Epsi_std), npmax(Epsi_std), t_ + 1) # ## Compute the Maximum Likelihood location and dispersion parameters p = (1 / t_) * ones((1, t_)) # probabilities mu_ML, sigma2_ML, _ = MaxLikelihoodFPLocDispT(Epsi, p, nu, threshold, last) # ## Compute the Maximum Likelihood pdf sigma_ML = sqrt(sigma2_ML) fML_eps = t.pdf((x - mu_ML) / sigma_ML, nu) # ## Compute the Maximum Likelihood cdf FML_eps = t.cdf((x - mu_ML) / sigma_ML, nu)
