def plotmap(self,fig,ax):
""" This function will plot the map of Alaska. The data will be plotted
over it and will use the basemap class to position everything.
Input
fig - The figure handle for the plots.
ax - The axes handle that the map will be plotted over.
Output
m - This is the handle for the basemap object.
"""
latlim2 = self.params['latbounds']
lonlim2 = self.params['lonbounds']
m = Basemap(projection='merc',lon_0=sp.mean(lonlim2),lat_0=sp.mean(latlim2),\
lat_ts=sp.mean(latlim2),llcrnrlat=latlim2[0],urcrnrlat=latlim2[1],\
llcrnrlon=lonlim2[0],urcrnrlon=lonlim2[1],\
rsphere=6371200.,resolution='i',ax=ax)
# draw coastlines, state and country boundaries, edge of map.
#m.drawcoastlines()
# m.drawstates()
# m.drawcountries()
m.readshapefile('st99_d00','states',drawbounds=True)
merstep = sp.round_((lonlim2[1]-lonlim2[0])/5.)
parstep = sp.round_((latlim2[1]-latlim2[0])/5.)
meridians=sp.arange(lonlim2[0],lonlim2[1],merstep)
parallels = sp.arange(latlim2[0],latlim2[1],parstep)
m.drawparallels(parallels,labels=[1,0,0,0],fontsize=10)
m.drawmeridians(meridians,labels=[0,0,0,1],fontsize=10)
plt.hold(True)
return m
def getCorrection(x, y, x_sv, y_sv):
"""looks up correction in calculated vector fields"""
xi = np.maximum(np.minimum(sp.round_(x / 100).astype('i'), x_sv.shape[0]),
0)
yi = np.maximum(np.minimum(sp.round_(y / 100).astype('i'), x_sv.shape[1]),
0)
return (x_sv[xi, yi], y_sv[xi, yi])
def deriva(t, z_ode, x, y, z):
v_deriva = sp.zeros(len(t))
for pos in range(len(t)):
v_deriva[pos] = sp.sqrt(
sp.dot((z_ode[pos, :3] - [x[pos], y[pos], z[pos]]),
(z_ode[pos, :3] - [x[pos], y[pos], z[pos]])))
v_d = v_deriva / 1000
dist_error = sp.sqrt(
sp.dot([x[pos], y[pos], z[pos]], [x[pos], y[pos], z[pos]]))
mdistancia = sp.round_(sp.amax(v_d), 1) # distancia máxima entre ambos
err = sp.round_(v_deriva[-1] / dist_error,
2) # % de error probando distintos N's
#print(f"Error = {err * 100}%")
#Gráfico P2
figure(1)
title(f"Distancia entre Eulerint y Odeint, d = {mdistancia} [km]")
xlabel("Tiempo, t [Horas]")
ylabel("Deriva, d [KM]")
plot(time, v_d, color="dodgerblue")
grid(False)
tight_layout()
savefig("P4.2.png")
def __expand_points(self, p0, p1, surff) -> list:
"""
ASCII drawing to come.
"""
rl = list() # return list
p0 = sp.array(p0)
p1 = sp.array(p1)
dist = sp.linalg.norm(p1[0:2] - p0[0:2]) # 2d distance only
# wont append starting point; is endpoint of previous segment;
# automatically avoids printing start coordinate to outupt
cz = surff(p0[0], p0[1]) # start point depth correction
# number of tick points to inspect
n = int(sp.ceil(dist/self[self.pt.xysampling]))
if n > 1:
# tick between start and end points
xt = sp.linspace(p0[0], p1[0], n, False)
yt = sp.linspace(p0[1], p1[1], n, False)
for i in range(1,n): # don't iterate over start/endpoints
zi = surff(xt[i], yt[i]) # if new depth correction is too large,
if abs(zi - cz) > self[self.pt.zthreshold]:
cz = zi
di = sp.linalg.norm(sp.array((xt[i], yt[i])) - p0[0:2])
z01= p0[2] + (p1[2]-p0[2])*(di/dist) # interpolate original depth
rl.append([float(sp.round_(i, self[self.pt.precision])) for i in [xt[i], yt[i], z01 + cz]])
# append end point
cz = surff(p1[0], p1[1]) # start point depth correction
rl.append([float(sp.round_(i, self[self.pt.precision])) for i in [p1[0], p1[1], p1[2] + cz]])
return rl
def _hough_transform(img, angles):
rows, cols = img.shape
# determine the number of bins
d = sp.ceil(sp.hypot(*img.shape))
nr_bins = 2 * d
bins = sp.linspace(-d, d, nr_bins)
# create the accumulator
out = sp.zeros((nr_bins, len(angles)), dtype=sp.float64)
# compute the sines/cosines
cos_theta = sp.cos(angles)
sin_theta = sp.sin(angles)
# constructe the x and y values
y = []
x = []
for i in xrange(rows):
y += [i] * cols
x += range(cols)
y = sp.array(y)
x = sp.array(x)
# flatten image
flattened_img = img.flatten()
for i, (c, s) in enumerate(zip(cos_theta, sin_theta)):
distances = x * c + y * s
bin_indices = (sp.round_(distances) - bins[0]).astype(sp.uint8)
bin_sums = sp.bincount(bin_indices, flattened_img)
out[:len(bin_sums), i] = bin_sums
return out
def makesumrule(ptype,plen,ts,lagtype='centered'):
""" This function will return the sum rule.
Inputs
ptype - The type of pulse.
plen - Length of the pulse in seconds.
ts - Sample time in seconds.
lagtype - Can be centered forward or backward.
Output
sumrule - A 2 x nlags numpy array that holds the summation rule.
"""
nlags = sp.round_(plen/ts)
if ptype.lower()=='long':
if lagtype=='forward':
arback=-sp.arange(nlags,dtype=int)
arforward = sp.zeros(nlags,dtype=int)
elif lagtype=='backward':
arback = sp.zeros(nlags,dtype=int)
arforward=sp.arange(nlags,dtype=int)
else:
arback = -sp.ceil(sp.arange(0,nlags/2.0,0.5)).astype(int)
arforward = sp.floor(sp.arange(0,nlags/2.0,0.5)).astype(int)
sumrule = sp.array([arback,arforward])
elif ptype.lower()=='barker':
sumrule = sp.array([[0],[0]])
return sumrule
def makepulse(ptype,plen,ts):
""" This will make the pulse array.
Inputs
ptype - The type of pulse used.
plen - The length of the pulse in seconds.
ts - The sampling rate of the pulse.
Output
pulse - The pulse array that will be used as the window in the data formation.
plen - The length of the pulse with the sampling time taken into account.
"""
nsamps = int(sp.round_(plen/ts))
if ptype.lower()=='long':
pulse = sp.ones(nsamps)
plen = nsamps*ts
elif ptype.lower()=='barker':
blen = sp.array([1,2, 3, 4, 5, 7, 11,13])
nsampsarg = sp.argmin(sp.absolute(blen-nsamps))
nsamps = blen[nsampsarg]
pulse = GenBarker(nsamps)
plen = nsamps*ts
#elif ptype.lower()=='ac':
else:
raise ValueError('The pulse type %s is not a valide pulse type.' % (ptype))
return (pulse,plen)
def real2discrete(ds, *real_value):
"""
이름 : real2discrete
기능 : 이 함수는 프로젝트 파일에서 real 공간 값으로 선언된 것을
전산모사에서 계산을 할 수 있도록 discrete 한 공간 값으로 변환해
준다. 따라서 주어지는 인자는 한 cell의 최소 단위인 dx가
주어지고, 변환하고자 하는 real 값들을 넣어준다.
인자 : dx, real_vals
반환 : discrete_val
보충 : 들어오는 real 값 개수에 맞추어 결과 discrete 값의 개수도
반환된다. 단지 한 개가 아닌 여러개 인자인 경우 tuple로 반환이
된다.
"""
number_members = len(real_value)
list_discrete_value = []
for mem in xrange(number_members):
array_real_value = sc.array(real_value[mem])
array_real_value = array_real_value/ds
discrete_value = sc.round_(array_real_value)
discrete_value = sc.array(discrete_value,'i')
list_discrete_value.append(discrete_value)
if len(list_discrete_value) == 1 :
return list_discrete_value[0]
else:
return list_discrete_value
def makesumrule(ptype,plen,ts):
nlags = sp.round_(plen/ts)
if ptype.lower()=='long':
arback = -sp.ceil(sp.arange(0,nlags/2.0,0.5)).astype(int)
arforward = sp.floor(sp.arange(0,nlags/2.0,0.5)).astype(int)
sumrule = sp.array([arback,arforward])
elif ptype.lower()=='barker':
sumrule = sp.array([[0],[0]])
return sumrule
def simulate_common_genotypes(n=1000, m=10000):
"""
Simulate genotypes
"""
snps = sp.random.random((m, n))
snps = sp.round_(snps)
snps = sp.array(snps, dtype='int8')
snps = snps[sp.sum(snps, 1) > 0]
print 'Done Simulating SNPs.'
return snps
def simulate_common_genotypes(n=1000, m=10000):
"""
Simulate genotypes
"""
snps = sp.random.random((m, n))
snps = sp.round_(snps)
snps = sp.array(snps, dtype='int8')
snps = snps[sp.sum(snps, 1) > 0]
print 'Done Simulating SNPs.'
return snps
def interpolate(self, canvas, status=None):
# Clear the interpolated canvas
canvas.interpolated = sp.zeros_like(canvas.fringes_image) - 1024.0
if status is not None:
status.set("Performing the interpolation", 70)
else:
print("Performing the interpolation")
# Iterate over all the triangles in the triangulation
for triangle in self.triangles:
# Create a shortcut to the triangle's vertices
co = triangle.vert_coordinates
# Calculate a few constants for the Barycentric Coordinates
# More info: https://codeplea.com/triangular-interpolation
div = (co[1, 0] - co[2, 0]) * (co[0, 1] - co[2, 1]) + (
co[2, 1] - co[1, 1]) * (co[0, 0] - co[2, 0])
a0 = (co[1, 0] - co[2, 0])
a1 = (co[2, 1] - co[1, 1])
a2 = (co[2, 0] - co[0, 0])
a3 = (co[0, 1] - co[2, 1])
# Calculate the bounds of a rectangle that fully encloses
# the current triangle
xmin = int(sp.amin(triangle.vert_coordinates[:, 1]))
xmax = int(sp.amax(triangle.vert_coordinates[:, 1])) + 1
ymin = int(sp.amin(triangle.vert_coordinates[:, 0]))
ymax = int(sp.amax(triangle.vert_coordinates[:, 0])) + 1
# Take out slices of the x and y arrays,
# containing the points' coordinates
x_slice = canvas.x[ymin:ymax, xmin:xmax]
y_slice = canvas.y[ymin:ymax, xmin:xmax]
# Use Barycentric Coordinates and the magic of numpy (scipy in this
# case) to perform the calculations with the C backend, instead
# of iterating on pixels with Python loops.
# If you have not worked with numpy arrays befor dear reader,
# the idea is that if x = [[0 1]
# [2 3]],
# then x*3+1 is a completely valid operation, returning
# x = [[1 4]
# [7 10]]
# Basically, we can do maths on arrays as if they were variables.
# Convenient, and really fast!
w0 = (a0 * (x_slice - co[2, 1]) + a1 * (y_slice - co[2, 0])) / div
w1 = (a2 * (x_slice - co[2, 1]) + a3 * (y_slice - co[2, 0])) / div
w2 = sp.round_(1 - w0 - w1, 10)
# Calculate the values for a rectangle enclosing our triangle
slice = (self.values[triangle.vertices[0]] * w0 +
self.values[triangle.vertices[1]] * w1 +
self.values[triangle.vertices[2]] * w2)
# Make a mask (so that we only touch the points
# inside of the triangle).
# In Barycentric Coordinates the points outside of the triangle
# have at least one of the coefficients negative, so we use that
mask = sp.logical_and(sp.logical_and(w0 >= 0, w1 >= 0), w2 >= 0)
# Change the points in the actual canvas
canvas.interpolated[ymin:ymax, xmin:xmax][mask] = slice[mask]
canvas.interpolation_done = True
def glmnetPlot(x, xvar = 'norm', label = False, ptype = 'coef', **options):
import matplotlib.pyplot as plt
# process inputs
xvar = getFromList(xvar, ['norm', 'lambda', 'dev'], 'xvar should be one of ''norm'', ''lambda'', ''dev'' ')
ptype = getFromList(ptype, ['coef', '2norm'], 'ptype should be one of ''coef'', ''2norm'' ')
if x['class'] in ['elnet', 'lognet', 'coxnet', 'fishnet']:
handle = plotCoef(x['beta'], [], x['lambdau'], x['df'], x['dev'],
label, xvar, '', 'Coefficients', **options)
elif x['class'] in ['multnet', 'mrelnet']:
beta = x['beta']
if xvar == 'norm':
norm = 0
nzbeta = beta
for i in range(len(beta)):
which = nonzeroCoef(beta[i])
nzbeta[i] = beta[i][which, :]
norm = norm + scipy.sum(scipy.absolute(nzbeta[i]), axis = 0)
else:
norm = 0
if ptype == 'coef':
ncl = x['dfmat'].shape[0]
if x['class'] == 'multnet':
for i in range(ncl):
mstr = 'Coefficients: Class %d' % (i)
handle = plotCoef(beta[i], norm, x['lambdau'], x['dfmat'][i,:],
x['dev'], label, xvar, '', mstr, **options)
if i < ncl - 1:
plt.figure()
else:
for i in range(ncl):
mstr = 'Coefficients: Response %d' % (i)
handle = plotCoef(beta[i], norm, x['lambdau'], x['dfmat'][i,:],
x['dev'], label, xvar, '', mstr, **options)
if i < ncl - 1:
plt.figure()
else:
dfseq = scipy.round_(scipy.mean(x['dfmat'], axis = 0))
coefnorm = beta[1]*0
for i in range(len(beta)):
coefnorm = coefnorm + scipy.absolute(beta[i])**2
coefnorm = scipy.sqrt(coefnorm)
if x['class'] == 'multnet':
mstr = 'Coefficient 2Norms'
handle = plotCoef(coefnorm, norm, x['lambdau'], dfseq, x['dev'],
label, xvar, '',mstr, **options);
else:
mstr = 'Coefficient 2Norms'
handle = plotCoef(coefnorm, norm, x['lambdau'], x['dfmat'][0,:], x['dev'],
label, xvar, '', mstr, **options);
return(handle)
def simulate_rare_genotypes(n=1000, m=10000, exp_scale=0.1):
"""
Simulating unlinked rare genotypes, rounding up exponentially distributed values.
m: number of causal variants
n: number of individuals
"""
snps = stats.expon.rvs(0, scale=exp_scale, size=(m, n))
snps = sp.round_(snps)
snps[snps > 1] = 1
snps = snps[sp.sum(snps, 1) > 0]
print 'Simulated %d dimorphic SNPs' % len(snps)
return snps
def doatest():
print("s1 is {}".format(sp.rad2deg(s1_aoa)))
print("s2 is {}".format(sp.rad2deg(s2_aoa)))
s1_est = music.Estimator(ants, music.covar(s1), nsignals=1)
s2_est = music.Estimator(ants, music.covar(s2), nsignals=1)
# s1
t1 = time()
s1_res = s1_est.doasearch()[0]
t1 = time() - t1
s1_err = sp.rad2deg(util.aoa_diff_rad(s1_res, s1_aoa))
#print("s1 estimated value: {} in {}s, error {} deg".format(sp.int_(sp.rad2deg(s1_res)),t1,s1_err))
print("s1 estimated value: {} in {}s, error {} deg".format(
sp.round_(sp.rad2deg(s1_res)), t1, s1_err))
#print("s1 estimated value: {} in {}s, error {} deg".format(sp.rad2deg(s1_res),t1,s1_err))
# s2
t2 = time()
s2_res = s2_est.doasearch()[0]
t2 = time() - t2
s2_err = sp.rad2deg(util.aoa_diff_rad(s2_res, s2_aoa))
print("s2 estimated value: {} in {}s, error {} deg".format(
sp.round_(sp.rad2deg(s2_res)), t2, s2_err))
# both signals
bothres = est.doasearch()
def simulate_rare_genotypes(n=1000, m=10000, exp_scale=0.1):
"""
Simulating unlinked rare genotypes, rounding up exponentially distributed values.
m: number of causal variants
n: number of individuals
"""
snps = stats.expon.rvs(0, scale=exp_scale, size=(m, n))
snps = sp.round_(snps)
snps[snps > 1] = 1
snps = snps[sp.sum(snps, 1) > 0]
print 'Simulated %d dimorphic SNPs' % len(snps)
return snps
def readMahalih5(filename,des_site):
""" This function will read the mahali GPS data into a GeoData data structure.
The user only has to give a filename and name of the desired site.
Input
filename - A string that holds the file name.
des_site - The site name. Should be listed in the h5 file in the
table sites.
"""
h5fn = Path(filename).expanduser()
with h5py.File(str(h5fn), "r", libver='latest') as f:
despnts = sp.where(f['data']['site']==des_site)[0]
# TODO: hard coded for now
doy = doy= f['data']['time'][despnts]
year = 2015*sp.ones_like(doy,dtype=int)
TEC = f['data']['los_tec'][despnts]
nTEC = f['data']['err_los_tec'][despnts]
vTEC = f['data']['vtec'][despnts]
az2sat = f['data']['az'][despnts]
el2sat = f['data']['az'][despnts]
piercelat = f['data']['pplat'][despnts]
piercelong = f['data']['pplon'][despnts]
satnum= f['data']['prn'][despnts]
recBias = f['data']['rec_bias'][despnts]
nrecBias = f['data']['err_rec_bias'][despnts]
# Make the integration time on the order of 15 seconds.
if (year==year[1]).all():
unixyear =(datetime(year[0],1,1,0,0,0,tzinfo=UTC) - EPOCH).total_seconds()
uttime = unixyear + sp.round_(24*3600*sp.column_stack((doy,doy+15./24./3600.))) # Making the difference in time to be a minute
else:
(y_u,y_iv) = np.unique(year,return_inverse=True)
unixyearu = sp.array([(datetime(iy,1,1,0,0,0,tzinfo=UTC) - EPOCH).total_seconds() for iy in y_u])
unixyear = unixyearu[y_iv]
uttime = unixyear + 24*3600*sp.column_stack((doy,doy+15./24./3600.))
data = {'TEC':TEC,'nTEC':nTEC,'vTEC':vTEC,'recBias':recBias,'nrecBias':nrecBias,'satnum':satnum,'az2sat':az2sat,'el2sat':el2sat}
coordnames = 'WGS84'
sensorloc = sp.nan*sp.ones(3)
dataloc = sp.column_stack((piercelat,piercelong, 350e3*sp.ones_like(piercelat)))
return (data,coordnames,dataloc,sensorloc,uttime)
def readMahalih5(filename,des_site):
""" This function will read the mahali GPS data into a GeoData data structure.
The user only has to give a filename and name of the desired site.
Input
filename - A string that holds the file name.
des_site - The site name. Should be listed in the h5 file in the
table sites.
"""
h5fn = Path(filename).expanduser()
with h5py.File(str(h5fn), "r", libver='latest') as f:
despnts = sp.where(f['data']['site']==des_site)[0]
# TODO: hard coded for now
doy = doy= f['data']['time'][despnts]
year = 2015*sp.ones_like(doy,dtype=int)
TEC = f['data']['los_tec'][despnts]
nTEC = f['data']['err_los_tec'][despnts]
vTEC = f['data']['vtec'][despnts]
az2sat = f['data']['az'][despnts]
el2sat = f['data']['az'][despnts]
piercelat = f['data']['pplat'][despnts]
piercelong = f['data']['pplon'][despnts]
satnum= f['data']['prn'][despnts]
recBias = f['data']['rec_bias'][despnts]
nrecBias = f['data']['err_rec_bias'][despnts]
# Make the integration time on the order of 15 seconds.
if (year==year[1]).all():
unixyear =(datetime(year[0],1,1,0,0,0,tzinfo=UTC) - EPOCH).total_seconds()
uttime = unixyear + sp.round_(24*3600*sp.column_stack((doy,doy+15./24./3600.))) # Making the difference in time to be a minute
else:
(y_u,y_iv) = np.unique(year,return_inverse=True)
unixyearu = sp.array([(datetime(iy,1,1,0,0,0,tzinfo=UTC) - EPOCH).total_seconds() for iy in y_u])
unixyear = unixyearu[y_iv]
uttime = unixyear + 24*3600*sp.column_stack((doy,doy+15./24./3600.))
data = {'TEC':TEC,'nTEC':nTEC,'vTEC':vTEC,'recBias':recBias,'nrecBias':nrecBias,'satnum':satnum,'az2sat':az2sat,'el2sat':el2sat}
coordnames = 'WGS84'
sensorloc = sp.nan*sp.ones(3)
dataloc = sp.column_stack((piercelat,piercelong, 350e3*sp.ones_like(piercelat)))
return (data,coordnames,dataloc,sensorloc,uttime)
def get_distribution(list):
"""Returns al que posible probability distributions for the given list"""
size = len(list)
x = scipy.arange(size)
y = scipy.int_(scipy.round_(scipy.stats.vonmises.rvs(5, size=size) * 255))
plt.figure(1)
h = plt.hist(y, bins=range(256), color='w')
dist_names = ['gamma', 'beta', 'rayleigh', 'norm', 'rayleigh']
for dist_name in dist_names:
dist = getattr(scipy.stats, dist_name)
param = dist.fit(y)
pdf_fitted = dist.pdf(x, *param[:-2], loc=param[-2],
scale=param[-1]) * size
plt.plot(pdf_fitted, label=dist_name)
plt.xlim(0, 255)
plt.legend(loc='upper right')
plt.savefig('distribuciones.png', bbox_inches='tight')
def simulate_genotypes(num_indivs=1000, num_snps=1000):
"""
Simulate genotypes
"""
snps = sp.random.random((num_snps, num_indivs))
snps = sp.round_(snps)
snps = sp.array(snps, dtype='int8')
snps = snps[sp.sum(snps, 1) > 0]
positions = []
chromosomes = []
for chrom in range(1, 6):
len_chrom = num_snps / 5
chromosomes.extend([chrom] * (len_chrom))
positions.extend(range(1, len_chrom + 1))
indiv_ids = range(num_indivs)
sd = snpsdata.construct_snps_data_set(snps, positions, chromosomes, indiv_ids)
print '\nDone Simulating SNPs.'
return sd
def makepulse(ptype,plen,ts):
nsamps = sp.round_(plen/ts)
if ptype.lower()=='long':
pulse = sp.ones(nsamps)
plen = nsamps*ts
elif ptype.lower()=='barker':
blen = sp.array([1,2, 3, 4, 5, 7, 11,13])
nsampsarg = sp.argmin(sp.absolute(blen-nsamps))
nsamps = blen[nsampsarg]
pulse = GenBarker(nsamps)
plen = nsamps*ts
#elif ptype.lower()=='ac':
else:
raise ValueError('The pulse type %s is not a valide pulse type.' % (ptype))
return (pulse,plen)
def simulate_genotypes(num_indivs=1000, num_snps=1000):
"""
Simulate genotypes
"""
snps = sp.random.random((num_snps, num_indivs))
snps = sp.round_(snps)
snps = sp.array(snps, dtype='int8')
snps = snps[sp.sum(snps, 1) > 0]
positions = []
chromosomes = []
for chrom in range(1, 6):
len_chrom = num_snps / 5
chromosomes.extend([chrom] * (len_chrom))
positions.extend(range(1, len_chrom + 1))
indiv_ids = range(num_indivs)
sd = snpsdata.construct_snps_data_set(snps, positions, chromosomes,
indiv_ids)
print '\nDone Simulating SNPs.'
return sd
def deriva(t, z_ode, z_eul):
v_deriva = sp.zeros(len(t))
for pos in range(len(t)):
v_deriva[pos] = sp.sqrt(
sp.dot((z_ode[pos, :3] - z_eul[pos, :3]),
(z_ode[pos, :3] - z_eul[pos, :3])))
v_d = v_deriva / 1000
mdistancia = sp.round_(sp.amax(v_d), 1) #distancia máxima entre ambos
#Gráfico P2
figure(2)
title(f"Distancia entre Eulerint y Odeint, d = {mdistancia} [km]")
xlabel("Tiempo, t [Horas]")
ylabel("Deriva, d [KM]")
plot(time, v_d, color="dodgerblue")
grid(False)
tight_layout()
savefig("P2.png")
def v2min_image_v(dr, cell, pbc=None, shifts_out=False):
"""
------
Authors: matscipy authors - https://github.com/libAtoms/matscipy
------
Apply minimum image convention to an array of distance vectors.
Parameters
----------
dr : array_like
Array of distance vectors.
cell : array_like, shape (n_dim,)
Cell extent in each direction.
pbc : array_like, optional, type bool
Periodic boundary conditions directions. Default is to
assume periodic boundaries in all directions.
Returns
-------
dr : array
Array of distance vectors, wrapped according to the minimum image
convention.
"""
# Check where distance larger than 1/2 cell. Particles have crossed
# periodic boundaries then and need to be unwrapped.
n_dim = len(cell)
rec = sp.diag(1. / sp.asarray(cell))
cell = sp.diag(sp.asarray(cell))
if pbc is not None:
rec *= sp.array(pbc, dtype=int).reshape(n_dim, 1)
dri = sp.round_(sp.dot(dr, rec))
shifts = sp.dot(dri, cell)
# Unwrap
if shifts_out:
return dr - shifts, shifts
else:
return dr - shifts
def AR_from_C(C, nc, max_order=10):
"""build covariance sequence for lags [0,max_order+1] from block toeplitz matrix
Here the input matrix C is assumed to exhibit a block structure with respect to the
multichanneled data and the temporal lag, where the temporal per channel forms the
Toeplitz structure, and the multichanneled part the block structure.
The output is a sequence of one nc x nc covariance matrix per lag in the toeplitz
structure of C. Also the AR coefficients for the order specified
"""
# inits
assert C.shape[0] == C.shape[1], 'C is not square'
tf = C.shape[0] / float(nc)
assert tf == sp.round_(tf), 'nc does not match tf'
tf, nc = int(tf), int(nc)
order = min(tf - 1, max_order + 1)
print 'doing lags upto:', order
gamma_y = sp.zeros((order, nc, nc))
GammaY0 = sp.empty(((order - 1) * nc, (order - 1) * nc))
# build gamma_y_seq from C
for o in xrange(order):
for c1 in xrange(nc):
for c2 in xrange(nc):
gamma_y[o, c1, c2] = C[c1 * tf + o, c2 * tf]
# build GammaY0 from gamma_y_seq, blockwise
for i in xrange(order - 1):
for j in xrange(i, order - 1):
GammaY0[i * nc:(i + 1) * nc, j * nc:(j + 1) * nc] = gamma_y[j - i]
if j > i:
GammaY0[j * nc:(j + 1) * nc,
i * nc:(i + 1) * nc] = gamma_y[j - i].T
# build calculation matrices
GammaY0inv = sp_la.inv(GammaY0)
gamma_y_seq = sp.hstack(gamma_y[1:])
# return
return sp.dot(gamma_y_seq, GammaY0inv), gamma_y
def trainModel(self, do_pca = False,out_dir='./cache', rftop = 40, class_labels = SP.array(['G1','S','G2M']), cv=10, npc=3, is_SVM=1, is_RFE=0 , scale=False):
if not os.path.exists(out_dir):
os.makedirs(out_dir)
CFG = {}
CFG['is_RFE'] = is_RFE # use recursive feature selection (can be slow for large datasets)
CFG['is_SVM'] = is_SVM # use SVM with univariate feature selection (faster than RFE)
CFG['CV_inner'] = cv #inner CV for RFE_CV: either an int or 'LOOCV'
CFG['out_dir'] = out_dir
CFG['do_pca'] = do_pca
CFG['lassotop'] = 20
self.cv = cv
Y = self.Y
labels = self.labels
var_names = self.geneNames
numClasses = self.numClasses
predRF = SP.zeros((len(labels),numClasses))
predSVM = SP.zeros((len(labels),numClasses))
predSVMrbf = SP.zeros((len(labels),numClasses))
predGNB = SP.zeros((len(labels),numClasses))
predLR = SP.zeros((len(labels),numClasses))
predLRall = SP.zeros((len(labels),numClasses))
names_dict={}
if self.cv == 'LOOCV':
loo = LeaveOneOut(len(labels))
CV_list = (list(iter(loo)))
CV_list.append((SP.array(range(Y.shape[0])), SP.array(range(Y.shape[0]))))#all data...
else:
skf = StratifiedKFold(labels, n_folds=self.cv)
CV_list = (list(iter(skf)))
CV_list.append((SP.array(range(Y.shape[0])), SP.array(range(Y.shape[0]))))#all data...
lambda_best = SP.zeros((1,len(CV_list))).ravel()
print("Performing cross validation ...")
for i in range(len(CV_list)):
if i<len(CV_list)-1:
print("Fold " + str(i+1) + " of " + str(len(CV_list)-1))
else:
print("Final model")
# string label for this fold
#get data of a CV run
cv_tr = CV_list[i][0]
cv_tst = CV_list[i][1]
lab_tr = labels[cv_tr]
Ytr = Y[cv_tr,:]
Ytst = Y[cv_tst,:]
lab_tst = labels[cv_tst]
if (i==len(CV_list)-1):
foldlabel = 'full'
if (self.Y_tst==None):
Ytst = Y[cv_tst,:]
lab_tst = labels[cv_tst]
else:
foldlabel = 'Test'
Ytst = self.Y_tst
lab_tst = self.labels_tst
else:
foldlabel = str(i)
if do_pca>=1:
npc = npc#3
#do PCA to get features
pcaCC = PCA(n_components=npc, whiten=False)
pcaCC.fit(Ytr)
pcaTst=pcaCC.transform(Ytst)
pcaTr=pcaCC.transform(Ytr)
#selection = SelectKBest(k=1)
#combined_features = FeatureUnion([("pca", pcaCC), ("univ_select", selection)])
combined_features = FeatureUnion([("pca", pcaCC)])
gnb = GaussianNB()
y_pred = gnb.fit(pcaTr, lab_tr).predict_proba(pcaTst)
if i<len(CV_list)-1:
predGNB[cv_tst,:] =y_pred#[:,1]
else:
predGNB_ts = y_pred#[:,1]
if do_pca==2:
Ytr = SP.concatenate((Ytr, pcaTr),1)
Ytst = SP.concatenate((Ytst, pcaTst),1)
pcnames = []
for pci in range(npc):
pcnames.append('PC'+str(pci+1))
var_names = SP.concatenate((var_names, SP.array(pcnames)),1)
print(" Computing random forest ...")
if CFG['is_RFE']==1:#Recursive feature selection with SVM
print(" Computing RFE with SVM ...")
svc = SVC(kernel="linear", probability=False, class_weight='auto')#use linear SVM for selection
rfecv = RFECV(estimator=svc, step=1,scoring='f1')
param_grid = dict(estimator__C=[0.1, 1, 10, 100, 1000])
clf_rfe = GridSearchCV(rfecv, param_grid=param_grid, cv=3, scoring='f1')#GridSearch to find optimal parameters
clf_rfe.fit(Ytr, lab_tr)
svc = SVC(kernel="linear", probability=False,C=clf_rfe.best_estimator_.estimator.C, class_weight='auto')#use linear SVM for selection
if CFG['CV_inner']=='':
rfecv = RFECV(estimator=svc, step=1,scoring='f1')
elif CFG['CV_inner']=='LOOCV':
rfecv = RFECV(estimator=svc, step=1,scoring='f1', cv=LeaveOneOut(len(lab_tr)))
else:
rfecv = RFECV(estimator=svc, step=1,scoring='f1', cv=StratifiedKFold(lab_tr, n_folds=CFG['CV_inner']))
clf_rfe.best_estimator_.fit(Ytr, lab_tr)
predicted = clf_rfe.best_estimator_.predict(Ytst)
if i<len(CV_list)-1:
predSVM[cv_tst,:] = predicted
else:
predSVM_ts[cv_tst] = predicted
classifier = svm.SVC(kernel='rbf', gamma=0.05, class_weight='auto', probability=True)#rbf kernel for prediction
param_grid = dict(C=[0.1, 1], gamma=[1e-1,1e-2,1e-3])
clf_rbf = GridSearchCV(classifier, param_grid=param_grid, cv=3, scoring='f1')
clf_rbf.fit(Ytr[:,clf_rfe.best_estimator_.ranking_==1], lab_tr)
clf_rbf.best_estimator_.fit(Ytr[:,clf_rfe.best_estimator_.ranking_==1], lab_tr)
predicted = clf_rbf.best_estimator_.predict_proba(Ytst[:,clf_rfe.best_estimator_.ranking_==1])
if i<len(CV_list)-1:
predSVMrbf[cv_tst,:] = predicted
fpr, tpr, thresholds = metrics.roc_curve(lab_tst, predicted[:,1])
if (i==len(CV_list)-1) | CFG["CV_plots"]>0:
PL.figure()
PL.plot(fpr, tpr)
PL.savefig(CFG['out_dir']+'/RF_SVM_'+foldlabel+'.pdf')
names_dict[foldlabel+'_SVM']=self.geneNames[clf_rfe.best_estimator_.ranking_==1]
elif CFG['is_SVM']==1:#univariate FS with rbf SVM; choose this if you hava a large data set (many features, eg RNAseq)
print(" SVM feature selection ...")
classifier = svm.SVC(kernel='rbf', gamma=0.05, class_weight='auto', probability=True)
selection = SelectKBest(k=1)
combined_features = FeatureUnion([("univ_select", selection)])
X_features = combined_features.fit(Ytr, lab_tr).transform(Ytr)
scaler = preprocessing.StandardScaler().fit(Ytr)
YtrS = scaler.transform(Ytr)
YtstS = scaler.transform(Ytst)
classifier.fit(X_features, lab_tr)
pipeline = Pipeline([("features", combined_features), ("svm", classifier)])
if CFG['do_pca']==3:
param_grid = dict(features__pca__n_components=SP.unique(SP.round_(SP.logspace(1.0,max(SP.log2(Ytr.shape[1]), SP.log2(10)),num=min(5,Ytr.shape[1]),base=2.0))),
features__univ_select__k=SP.unique(SP.round_(SP.logspace(3.0,SP.log2(Ytr.shape[1]),num=min(10,Ytr.shape[1]),base=2.0))),
svm__C=[0.1, 1, 10], svm__gamma=[1e-1,1e-2,1e-3])
else:
C_range = 10. ** SP.arange(0, 2)
gamma_range = 10. ** SP.arange(-5, 1)
param_grid = dict(features__univ_select__k=SP.unique(SP.round_(SP.logspace(3.0,SP.log2(Ytr.shape[1]),num=min(10,Ytr.shape[1]),base=2.0))),
svm__C=C_range, svm__gamma=gamma_range)
clf = GridSearchCV(pipeline, param_grid=param_grid, cv=5, scoring='f1')
clf.fit(YtrS, lab_tr)
print("The best classifier is: ", clf.best_estimator_)
select_best=clf.best_estimator_.get_params()['features__univ_select']
#names_dict[foldlabel+'_SVM']=self.geneNames[SP.argsort(-1.0*select_best.scores_)[0:(select_best.k-1)]]
expected = lab_tst
predicted = clf.best_estimator_.predict_proba(YtstS)
if i<len(CV_list)-1:
predSVM[cv_tst,:] = predicted
else:
predSVM_ts = predicted
#print(clf.best_estimator_)
classifier = svm.SVC(kernel='rbf', gamma=0.05, class_weight='auto', probability=True)#rbf kernel for prediction
param_grid = dict(C=[1,10], gamma=[ 1e-1,1e-2,1e-3])
clf_rbf = GridSearchCV(classifier, param_grid=param_grid, cv=5, scoring='f1')
clf_rbf.fit(Ytr, lab_tr)
clf_rbf.best_estimator_.fit(Ytr, lab_tr)
predicted = clf_rbf.best_estimator_.predict_proba(Ytst)
if i<len(CV_list)-1:
predSVMrbf[cv_tst,:] = predicted
else:
predSVMrbf_ts = predicted
#do lasso with regularisation path
cs = l1_min_c(Ytr, lab_tr, loss='log') * SP.logspace(0, 3)
print(" Computing regularization path ...")
lasso = linear_model.LogisticRegression(C=cs[0]*10.0, penalty='l1', tol=1e-6)
param_grid = dict(C=cs)
clf_lr = GridSearchCV(lasso, param_grid=param_grid, cv=5, scoring='f1')
clf_lr.fit(Ytr, lab_tr)
clf_lr.best_estimator_.fit(Ytr, lab_tr)
lambda_best[i] = clf_lr.best_params_.get('C')
predicted = clf_lr.best_estimator_.predict_proba(Ytst)
clf = linear_model.LogisticRegression(C=cs[0]*10.0, penalty='l1', tol=1e-6)
coefs_ = []
for c in cs:
clf.set_params(C=c)
clf.fit(Ytr, lab_tr)
coefs_.append(clf.coef_.ravel().copy())
if i<len(CV_list)-1:
predLR[cv_tst,:] = predicted
else:
predLR_ts = predicted
coefs_ = SP.array(coefs_)
# get ordering by importance (how many times they appear)
order=(coefs_!=0).sum(axis=0).argsort()
order=order[::-1] # descending
# store this order
featrank_lasso = order
showtop= min(Ytr.shape[1], CFG['lassotop'])
clfAll = linear_model.LogisticRegression(C=1e5, penalty='l2', tol=1e-6)
clfAll.fit(Ytr, lab_tr)
predicted = clfAll.predict_proba(Ytst)
if i<len(CV_list)-1:
predLRall[cv_tst,:] = predicted
else:
predLRall_ts = predicted
forest = ExtraTreesClassifier(n_estimators=500,
random_state=0, criterion="entropy", bootstrap=False)
#forest = RandomForestClassifier(n_estimators=500,
# random_state=0, criterion="entropy")
forest.fit(Ytr, lab_tr)
pred = forest.predict_proba(Ytst)
#pdb.set_trace()
if i<len(CV_list)-1:
predRF[cv_tst,:] = pred#[:,1]
else:
predRF_ts = pred#[:,1]
importances = forest.feature_importances_
std = SP.std([tree.feature_importances_ for tree in forest.estimators_],
axis=0)
topfeat=min(Ytr.shape[1], rftop)
indices = SP.argsort(importances)[::-1][0:topfeat]
# store full feature ranking
featrank_rf = SP.argsort(importances)[::-1]
# Plot the feature importances of the forest
if (i==len(CV_list)-1):
PL.figure()
#PL.title("Feature importances, Fold "+foldddPPlabel+', AUC='+str(SP.round_(metrics.auc(fpr, tpr),3)))
PL.title("Feature importances")
#PL.bar(range(topfeat), importances[indices],color="r", yerr=std[indices], align="center")
PL.bar(range(topfeat), importances[indices],color="r", align="center")
PL.xticks(range(topfeat), indices, rotation=70)
PL.gca().set_xticklabels(var_names[indices])
PL.setp(PL.gca().get_xticklabels(), fontsize=8)
PL.xlim([-1, topfeat])
PL.savefig(out_dir+'/RF_featureimportance_'+foldlabel+'.pdf')
f2 = open(os.path.join(out_dir,'classification_reportCV.txt') ,'w')
predRFv = SP.argmax(predRF_ts,axis=1)+1
predRF_trv = SP.argmax(predRF,axis=1)+1
self.scores = predRF
self.scores_tst = predRF_ts
self.ranking = var_names[indices]
predLRv = SP.argmax(predLR_ts,axis=1)+1
predLR_trv = SP.argmax(predLR,axis=1)+1
self.scoresLR = predLR
self.scoresLR_tst = predLR_ts
predLRallv = SP.argmax(predLRall_ts,axis=1)+1
predLRall_trv = SP.argmax(predLRall,axis=1)+1
self.scoresLRall = predLRall
self.scoresLRall_tst = predLRall_ts
if CFG['is_SVM']==1:
predSVMv = SP.argmax(predSVM_ts,axis=1)+1
predSVM_trv = SP.argmax(predSVM,axis=1)+1
self.scoresSVM = predSVM
self.scoresSVM_tst = predSVM_ts
predSVMrbfv = SP.argmax(predSVMrbf_ts,axis=1)+1
predSVMrbf_trv = SP.argmax(predSVMrbf,axis=1)+1
self.scoresSVMrbf = predSVMrbf
self.scoresSVMrbf_tst = predSVMrbf_ts
predGNBv = SP.argmax(predGNB_ts,axis=1)+1
predGNB_trv = SP.argmax(predGNB,axis=1)+1
self.scoresGNB = predGNB
self.scoresGNB_tst = predGNB_ts
print("Classification report for classifier %s:\n%s\n" % ('Gaussian Naive Bayes', metrics.classification_report(self.labels, predGNB_trv)))
print("Classification report for classifier %s:\n%s\n" % ('Random Forest', metrics.classification_report(self.labels, predRF_trv)))
print("Classification report for classifier %s:\n%s\n" % ('LR', metrics.classification_report(self.labels, predLR_trv)))
print("Classification report for classifier %s:\n%s\n" % ('LRall', metrics.classification_report(self.labels, predLRall_trv)))
if CFG['is_RFE']==1:
print("Classification report for classifier %s:\n%s\n" % ('SVM ', metrics.classification_report(labels, predSVM>0.5)),file=f2)
elif CFG['is_SVM']==1:
print("Classification report for classifier %s:\n%s\n" % ('SVM', metrics.classification_report(self.labels, predSVM_trv)))
print("Classification report for classifier %s:\n%s\n" % ('SVMrbf', metrics.classification_report(self.labels, predSVMrbf_trv)))
f2.close()
def plotPerformance(self,perfType = 'ROC',plot_test=True, out_dir = './cache',class_labels = ['G1 phase','S phase','G2M phase'], method = 'RF'):
plparams = {'backend': 'pdf',
'axes.labelsize': 14,
'text.fontsize': 14,
'legend.fontsize': 13,
'xtick.labelsize': 14,
'ytick.labelsize': 14,
'text.usetex': False}
PL.rcParams.update(plparams)
assert(perfType in ['ROC', 'PR'])
if plot_test ==True:
if method=='RF':
labs = self.labels_tst
scores = self.scores_tst
elif method=='LR':
labs = self.labels_tst
scores = self.scoresLR_tst
elif method=='LRall':
labs = self.labels_tst
scores = self.scoresLRall_tst
elif method=='GNB':
labs = self.labels_tst
scores = self.scoresGNB_tst
elif method=='SVM':
labs = self.labels_tst
scores = self.scoresSVM_tst
elif method=='SVMrbf':
labs = self.labels_tst
scores = self.scoresSVMrbf_tst
else:
if method=='RF':
labs = self.labels
scores = self.scores
elif method=='LR':
labs = self.labels
scores = self.scoresLR
elif method=='LRall':
labs = self.labels
scores = self.scoresLRall
elif method=='GNB':
labs = self.labels
scores = self.scoresGNB
elif method=='SVM':
labs = self.labels
scores = self.scoresSVM
elif method=='SVMrbf':
labs = self.labels
scores = self.scoresSVMrbf
PL.figure()
#col = ['r', 'b', 'g']
col = ['#1b9e77', '#d95f02', '#7570b3']
aucList = SP.zeros((scores.shape[1],))
for ind in range(scores.shape[1]):
labels_i = labs.copy()
scores_i = scores[:,ind]
labels_i[labs==ind+1] = 1
labels_i[labs!=ind+1] = 0
if perfType == 'ROC':
fpr, tpr, thresholds = metrics.roc_curve(labels_i, scores_i)
aucList[ind] = metrics.auc(fpr, tpr)
elif perfType == 'PR':
fpr, tpr, thresholds = metrics.precision_recall_curve(labels_i, scores_i)
PL.plot(fpr, tpr, '-', c=col[ind])
if perfType=='ROC':
PL.title("ROC curve ccClassify")
PL.xlabel('FPR')
PL.ylabel('TPR')
leg_str = list()
for i in range(len(class_labels)):
leg_str.append(class_labels[i]+', AUC = '+str(SP.round_(aucList[i],3)))
PL.legend(leg_str, loc='lower-right')
ax = plt.gca()
ax.spines["right"].set_visible(False)
ax.spines["top"].set_visible(False)
ax.get_xaxis().tick_bottom()
ax.get_yaxis().tick_left()
PL.savefig(out_dir+'/ROC_test'+str(plot_test)+'_'+method+'.pdf',bbox_inches='tight')
else:
PL.title("PR curve ccClassify")
PL.xlabel('Precision')
PL.ylabel('Recall')
leg_str = list()
for i in range(len(class_labels)):
leg_str.append(class_labels[i])#+', AUC = '+str(SP.round_(aucList[i],3)))
PL.legend(leg_str, loc='lower-right')
ax = plt.gca()
ax.spines["right"].set_visible(False)
ax.spines["top"].set_visible(False)
ax.get_xaxis().tick_bottom()
ax.get_yaxis().tick_left()
PL.savefig(out_dir+'/ROC_test'+str(plot_test)+'_'+method+'.pdf',bbox_inches='tight')
def refresh_table(request, option_name):
try:
#print " ------------------------------------------------------------------------------ "
#print request.POST
#print " ------------------------------------------------------------------------------ "
#print request.is_ajax()
#print " ------------------------------------------------------------------------------ "
if request.is_ajax() and request.POST:
option_name = request.POST['option_name']
options = OptionDefinition.objects.all()
option = [o for o in options if o.name == option_name][0]
optioncontracts = sorted(
OptionContract.objects.filter(optiondefinition_id=option.id),
key = lambda x : x.strike)
published_options = sorted(
PublishOptionContract.objects.filter(optiondefinition_id=option.id),
key = lambda x : x.strike)
future = option.future
today = datetime.date.today()
time2expiry = hf.diff_dates_year_fraction(option.expiry_date, today)
# we need to know the last vol to calculate the change, we store this in a dict
# accessed by strike value
published_options_dict = dict([(i.strike, i.vol) for i in published_options])
published_options_changes_dict = dict([(i.strike, i.change) for i in published_options])
strikes = [float(i) for i in request.POST.getlist('strikes[]')]
#json_strikes = json.dumps(strikes)
vols = [float(i) for i in request.POST.getlist('vols[]')]
call_values = [hf.black_pricer(time2expiry, future.bid, K, v, call = True) for K, v in zip(strikes, vols)]
put_values = [hf.black_pricer(time2expiry, future.bid, K, v, call = False) for K, v in zip(strikes, vols)]
call_deltas = [hf.black_delta(time2expiry, future.bid, K, v, call = True) for K, v, in zip(strikes, vols)]
# extra stuff Gareth wants
put_delta = [hf.black_delta(time2expiry, future.bid, K, vol, call = False) for K, vol in zip(strikes, vols)]
value_gamma = [hf.black_gamma(time2expiry, future.bid, K, vol) for K, vol in zip(strikes, vols)]
value_theta = [hf.black_theta(time2expiry, future.bid, K, vol) for K, vol in zip(strikes, vols)]
value_vega = [hf.black_vega(time2expiry, future.bid, K, vol) for K, vol in zip(strikes, vols)]
changes = []
# if we're updating on a publish table event we just want the changes from the db, this is because
# the db gets changed before we update the table, therefore when we query the db to find what the
# old vols were, because we've just changed them, they're the same as the new.
if request.POST['update'] == 'gradeX':
for s, v in zip(strikes, vols):
changes.append(published_options_changes_dict[s])
# if we're updated on an update table event, then we need to calculate the changes
else:
for s, v in zip(strikes, vols):
if s in published_options_dict:
changes.append(v - published_options_dict[s])
else:
changes.append(0.0)
#print changes
heading_row = [future.name, option.name, future.bid]
atm_strike = hf.ATM_strike(strikes, future.bid)
random_column = []
for pos, (v, s) in enumerate(zip(vols, strikes)):
if s == atm_strike:
random_column.append(scipy.round_(hf.black_pricer(time2expiry, future.bid, s, v, call = True) + hf.black_pricer(time2expiry, future.bid, s, v, call = False), decimals = 5))
else:
random_column.append('')
return HttpResponse(json.dumps({
'strikes' : strikes,
'call_values' : [scipy.round_(i, decimals=5) for i in call_values],
'put_values' : [scipy.round_(i, decimals=5) for i in put_values],
'call_deltas' : [scipy.round_(i, decimals=5) for i in call_deltas],
'put_deltas' : [scipy.round_(i, decimals=5) for i in put_delta],
'value_gamma' : [scipy.round_(i, decimals=5) for i in value_gamma],
'value_theta' : [scipy.round_(i, decimals=5) for i in value_theta],
'value_vega' : [scipy.round_(i, decimals=5) for i in value_vega],
'vols' : [scipy.round_(i, decimals=5) for i in vols],
'changes' : [scipy.round_(i, decimals=5) for i in changes],
'random_column' : random_column,
'heading_row' : heading_row,
}), mimetype="application/json" )
else:
#print "else raise"
raise Http404
except Exception as e:
#print e
raise Http404
# -*- coding: utf-8 -*- """ Created on Wed Jun 19 00:00:01 2013 ╛ @author: jhinebaugh """ import OpenPNM from time import clock import scipy as sp clock() print 'Creating 25,000 pore cubic network' cubic = OpenPNM.Geometry.Cubic().generate(domain_size=[50,50,10],lattice_spacing=[1.0]) print 'Finished at time =',sp.round_(clock(),2),'seconds' print '' print 'Creating 100 pore Delauny network' delaunay = OpenPNM.Geometry.Delaunay().generate(domain_size=[10,10,10],num_pores=100) print 'Finished at time =',sp.round_(clock(),2),'seconds' print '' print 'Creating #### pore imported network' matfile = OpenPNM.Geometry.MatFile().generate(filename='standard_cubic_5x5x5.mat') print matfile print 'Finished at time =',sp.round_(clock(),2),'seconds' print '' print "Adding 'air' and 'water' for each network type" air_cubic = OpenPNM.Fluids.Air().create(fluid_name='air_cubic') water_cubic = OpenPNM.Fluids.Water().create(fluid_name='water_cubic') air_delaunay = OpenPNM.Fluids.Air().create(fluid_name='air_delaunay') water_delaunay = OpenPNM.Fluids.Water().create(fluid_name='water_delaunay') air_matfile = OpenPNM.Fluids.Air().create(fluid_name='air_matfile') water_matfile = OpenPNM.Fluids.Water().create(fluid_name='water_matfile')
def option(request, option_name):
try:
options = OptionDefinition.objects.all()
option = [o for o in options if o.name == option_name][0]
optioncontracts = sorted(
OptionContract.objects.filter(optiondefinition_id=option.id),
key = lambda x : x.strike)
strikes = [o.strike for o in optioncontracts]
bids = [o.bid if o.bid > 0 else None for o in optioncontracts]
json_strikes = json.dumps(strikes)
json_bids = json.dumps(bids)
asks = [o.ask if o.ask > 0 else None for o in optioncontracts]
json_asks = json.dumps(asks)
values = [o.value if o.value > 0 else None for o in optioncontracts]
json_values = json.dumps(values)
bid_volume = json.dumps([o.bid_volume if o.bid_volume > 0 else None for o in optioncontracts])
ask_volume = json.dumps([o.ask_volume if o.ask_volume > 0 else None for o in optioncontracts])
last_trade_value = [o.last_trade_value if o.last_trade_value > 0 else None for o in optioncontracts]
json_last_trade_value = json.dumps(last_trade_value)
last_trade_volume = json.dumps([o.last_trade_volume if o.last_trade_volume > 0 else None for o in optioncontracts])
last_updated = json.dumps([o.last_updated.strftime("%Y-%m-%d %H:%M:%S") for o in optioncontracts])
#get future price
future = option.future
callbool = [True if o.easy_screen_mnemonic[-3] == 'c' else False for o in optioncontracts]
today = datetime.date.today()
time2expiry = hf.diff_dates_year_fraction(option.expiry_date, today)
# calc vols and deltas
bid_vols = [hf.black_pricer_vol(time2expiry, future.bid, K, Val, call) for K, Val, call in zip(strikes, bids, callbool)]
json_bid_vols = json.dumps(bid_vols)
bid_delta = [hf.black_delta(time2expiry, future.bid, K, vol, call) for K, vol, call in zip(strikes, bid_vols, callbool)]
json_bid_delta = json.dumps(bid_delta)
# always future.bid. Why? you can't hedge against a price future price that don't exist, so we don't use the value. We
# therefore need to choose one of bid or ask. We choose bid.
ask_vols = [hf.black_pricer_vol(time2expiry, future.bid, K, Val, call) for K, Val, call in zip(strikes, asks, callbool)]
json_ask_vols = json.dumps(ask_vols)
# always future.bid. Why? you can't hedge against a price future price that don't exist, so we don't use the value. We
# therefore need to choose one of bid or ask. We choose bid.
ask_delta = json.dumps([hf.black_delta(time2expiry, future.bid, K, vol, call) for K, vol, call in zip(strikes, ask_vols, callbool)])
# always future.bid. Why? you can't hedge against a price future price that don't exist, so we don't use the value. We
# therefore need to choose one of bid or ask. We choose bid.
value_vols = [hf.black_pricer_vol(time2expiry, future.bid, K, Val, call) for K, Val, call in zip(strikes, values, callbool)]
#interpolate on vol surface. Presently order 3 spline.
value_vols = hf.spline_interpolate(value_vols, strikes)
json_value_vols = json.dumps(value_vols)
# always future.bid. Why? you can't hedge against a price future price that don't exist, so we don't use the value. We
# therefore need to choose one of bid or ask. We choose bid.
value_delta = [hf.black_delta(time2expiry, future.bid, K, vol, call) for K, vol, call in zip(strikes, value_vols, callbool)]
json_value_delta = json.dumps(value_delta)
# always future.bid. Why? you can't hedge against a price future price that don't exist, so we don't use the value. We
# therefore need to choose one of bid or ask. We choose bid.
call_delta = [hf.black_delta(time2expiry, future.bid, K, vol, call = True) for K, vol in zip(strikes, value_vols)]
last_trade_vols = [hf.black_pricer_vol(time2expiry, future.last_trade_value, K, Val, call) for K, Val, call in zip(strikes, last_trade_value, callbool)]
json_last_trade_vols = json.dumps(last_trade_vols)
last_trade_delta = json.dumps([hf.black_delta(time2expiry, future.last_trade_value, K, vol, call) for K, vol, call in zip(strikes, last_trade_vols, callbool)])
# extra stuff Gareth wants
put_delta = [hf.black_delta(time2expiry, future.bid, K, vol, call = False) for K, vol in zip(strikes, value_vols)]
value_gamma = [hf.black_gamma(time2expiry, future.bid, K, vol) for K, vol in zip(strikes, value_vols)]
value_theta = [hf.black_theta(time2expiry, future.bid, K, vol) for K, vol in zip(strikes, value_vols)]
value_vega = [hf.black_vega(time2expiry, future.bid, K, vol) for K, vol in zip(strikes, value_vols)]
publishedcontracts = sorted(
PublishOptionContract.objects.filter(optiondefinition_id=option.id),
key = lambda x : x.strike)
if len(publishedcontracts) == 0:
published = False
published_time = json.dumps(datetime.datetime(1900,1,1,0,0,0).strftime("%Y-%m-%d %H:%M:%S"))
table_data = []#(future.name, option.month_tag, future.bid, '', '', '', '', '')]
atm_strike = hf.ATM_strike(strikes, future.bid)
random_column = []
for pos, (v, s) in enumerate(zip(value_vols, strikes)):
if s == atm_strike:
random_column.append(scipy.round_(hf.black_pricer(time2expiry, future.bid, s, v, call = True) + hf.black_pricer(time2expiry, future.bid, s, v, call = False), decimals = 5))
else:
random_column.append('')
for p in xrange(len(random_column)):
table_data.append(
(
scipy.round_(strikes[p], decimals=5),
scipy.round_(hf.black_pricer(time2expiry, future.bid, strikes[p], value_vols[p], True), decimals=5),
scipy.round_(hf.black_pricer(time2expiry, future.bid, strikes[p], value_vols[p], False), decimals=5),
random_column[p],
scipy.round_(call_delta[p], decimals=5),
scipy.round_(put_delta[p], decimals=5),
scipy.round_(value_gamma[p], decimals=5),
scipy.round_(value_theta[p], decimals=5),
scipy.round_(value_vega[p], decimals=5),
scipy.round_(value_vols[p], decimals=5),
'-'
)
)
else:
# here we need to get stuff from the publishedcontracts
published = True
published_time = publishedcontracts[0].publish_time
table_data = []#(future.name, option.month_tag, publishedcontracts[0].future_value, '', '', '', '', '')]
atm_strike = hf.ATM_strike([c.strike for c in publishedcontracts], publishedcontracts[0].future_value)
time2expiry = hf.diff_dates_year_fraction(option.expiry_date, publishedcontracts[0].publish_time)
random_column = []
for pos, o in enumerate(publishedcontracts):
if o.strike == atm_strike:
random_column.append(scipy.round_(hf.black_pricer(time2expiry, o.future_value, o.strike, o.vol, call = True) + hf.black_pricer(time2expiry, o.future_value, o.strike, o.vol, call = False), decimals = 5))
else:
random_column.append('')
for pos, c in enumerate(publishedcontracts):
table_data.append(
(
scipy.round_(c.strike, decimals=5),
scipy.round_(c.call_value, decimals=5),
scipy.round_(c.put_value, decimals=5),
random_column[pos],
scipy.round_(c.call_delta, decimals=2),
scipy.round_(c.put_delta, decimals=2),
scipy.round_(c.gamma, decimals=3),
scipy.round_(c.theta, decimals=2),
scipy.round_(c.vega, decimals=2),
scipy.round_(c.vol, decimals=5),
scipy.round_(c.change, decimals=5),
)
)
except Exception as e:
#print e
#raise Http404
return render(
request, 'marketdata/no-data-option.html',
{
'option': option,
})
return render(
request, 'marketdata/option.html',
{
'future': future,
'future_bid': future.bid,
'future_last_updated': future.last_updated,
'option': option,
'strikes' : json_strikes,
'non_json_strikes' : strikes,
'bids' : json_bid_vols,
'bids_delta' : json_bid_delta,
'asks' : json_ask_vols,
'asks_delta' : ask_delta,
'values' : json_value_vols,
'values_delta' : json_value_delta,
'bid_volume' : bid_volume,
'ask_volume' : ask_volume,
'last_trade_value' : json_last_trade_vols,
'last_trade_volume' : last_trade_volume,
'last_updated' : last_updated,
'published' : published,
'published_time' : published_time,
'table_data' : table_data
}
)
def get_ramp(x0, x1, vmax, a, dt, output='ramp only'):
"""
Generate a ramp trajectory from x0 to x1 with constant
acceleration, a, to maximum velocity v_max.
Note, the main purlpose of this routine is to generate a
trajectory from x0 to x1. For this reason v_max and a are adjusted
slightly to work with the given time step.
Arguments:
x0 = starting position
x1 = ending position
vmax = maximum allowed velocity
a = constant acceleration
Keywords:
output = 'ramp only' or 'full'
when ramp only is selected then only the velocity ramp is returned.
If 'full' is selected the adjusted acceleration and maximum velocity
are also returned.
Ouput:
ramp = ramp trajectory form x0 to x1
"""
# Insure we are dealing with floating point numbers
x0, x1 = float(x0), float(x1)
vmax, a = float(vmax), float(a)
dt = float(dt)
vmax, a = abs(vmax), abs(a) # Make sure that v_max and a are positive
# Check to see if there is anything to do
if x0 == x1:
return scipy.array([x0])
# Get distance and sign indicating direction
dist = abs(x1 - x0)
sign = scipy.sign(x1 - x0)
# Determine if we will reach v_max
t2vmax = vmax / a
t2halfdist = scipy.sqrt(0.5 * dist / a)
if t2vmax > t2halfdist:
# Trajectory w/o constant velocity segment
T = scipy.sqrt(dist / a)
n = int(scipy.round_((1.0 / dt) * T))
# Adjust accel and duration for rounding of n (discrete time steps)
a = dist / (n * dt)**2
T = scipy.sqrt(dist / a)
# Generate trajectory
t = scipy.linspace(0.0, 2.0 * T, 2 * n + 1)
def f1(t):
return 0.5 * sign * a * (t**2)
def f2(t):
s = t - T
return f1(T) + sign * a * T * s - 0.5 * sign * a * s**2
func_list = [f1, f2]
cond_list = [t <= T, t > T]
ramp = x0 + scipy.piecewise(t, cond_list, func_list)
else:
# Trajectory w/ constant velocity segment
# Compute acceleration time and adjust acceleration
T1 = vmax / a
n = int(scipy.round_(T1 / dt))
a = vmax / (n * dt) # Adjusted acceleration
T1 = vmax / a # Adjusted acceleration time
# Compute and adjust constant velocity time
T2 = dist / vmax - T1
m = int(scipy.round_(T2 / dt))
vmax = dist / (dt * (n + m)) # Adjusted max velocity
T2 = dist / vmax - T1 # Adjusted constant velocity time
# Generate trajectory
t = scipy.linspace(0.0, 2.0 * T1 + T2, 2 * n + m + 1)
def f1(t):
return 0.5 * sign * a * (t**2)
def f2(t):
s = t - T1
return f1(T1) + sign * vmax * s
def f3(t):
s = t - T1 - T2
return f2(T1 + T2) + sign * vmax * s - 0.5 * sign * a * s**2
func_list = [f1, f2, f3]
cond_list = [
t <= T1,
scipy.logical_and(t > T1, t <= T1 + T2), t > T1 + T2
]
ramp = x0 + scipy.piecewise(t, cond_list, func_list)
if output == 'ramp only':
return ramp
elif output == 'full':
return ramp, vmax, a
else:
raise ValueError, 'unknown keyword option output=%s' % (output, )
print '\nwithout using cache'
print tn, '(*n)'
print '\n2nd\t4th\t6th\t8th\t(spatial order)'
print '1D'
for n, nx in [(256, 256), (512, 512)]:
print '(%d=%d)' % (n, nx)
n2 = 6 * n + 3 * n + (2 * n + 1) + (2 * n + 1)
n4 = 6 * n + 7 * n + (4 * n + 3) + (4 * n + 3)
n6 = 6 * n + 11 * n + (6 * n + 5) + (6 * n + 5)
n8 = 6 * n + 15 * n + (8 * n + 7) + (8 * n + 7)
#rn = 6.*n + (2*p-1)*n + (p*n+p-1) + (p*n+p-1)
rn = 6. * n + (4 * p - 1) * n + 2 * (p - 1)
#print rn
ratio = rn / (tn * n) * 100
print sc.round_(ratio, 2), '%'
print '\n2D'
for n, nx, ny in [(256, 16, 16), (512, 32, 16)]:
print '(%d=%dx%d)' % (n, nx, ny)
n2 = 6 * n + (2 * n + nx) + (2 * n + ny) + (n + (nx + ny))
n4 = 6 * n + (4 * n + 3 * nx) + (4 * n + 3 * ny) + (n + 3 * (nx + ny))
#rn = 6.*n + (p*n+(p-1)*nx) + (p*n+(p-1)*ny) + (n+(p-1)*(nx+ny))
rn = 6. * n + (2 * p + 1) * n + 2 * (p - 1) * (nx + ny)
ratio = rn / (tn * n) * 100
print sc.round_(ratio, 2), '%'
print '\n3D'
for n, nx, ny, nz in [(256, 16, 4, 4), (512, 16, 8, 4)]:
print '(%d=%dx%dx%d)' % (n, nx, ny, nz)
n2 = 6 * n + 3 * n + 2 * (nx * ny + ny * nz + nz * nx)
def __pyramid_ski__(frame,t = 7,k = 2):
# create image pyramid from img, skimage implemetation
py = pyramid_gaussian(frame, downscale = k)
return [sp.round_(255.0*e).astype(sp.uint8) for e in py][:t]
def dnToTOA(inPath, outPath, sensor):
"""
Convertir les DNs en Radiance et/ou réflectance des images PlanetScope et RapidEye
sensor = 0 if PlanetScope or 1 if RapidEye
"""
if sensor == 0: # PlanetScope
# Search MS Image and Metadata XML File
lstFile = [file for file in os.listdir(inPath)]
# print (lstFile)
for file in lstFile:
if file.split('_')[-1] == "AnalyticMS.tif":
inRaster = file
elif file.split('_')[-1] == "metadata.xml":
inMetadata = file
# print (inRaster,inMetadata)
# Parse XML and Get reflectanceCoefficient from Metadata File
tree = ET.parse(os.path.join(inPath, inMetadata))
lstBand = []
lstRCoef = []
dicMetadata = {}
for element in tree.iter():
if re.match("(.*)bandNumber", element.tag):
lstBand.append(int(element.text))
elif re.match("(.*)reflectanceCoefficient", element.tag):
lstRCoef.append(float(element.text))
# print (lstBand, lstRCoef)
for i in range(len(lstBand)):
dicMetadata.update({lstBand[i]: lstRCoef[i]})
# print (dicMetadata)
# Open Raster File and Convert DN to TOA Reflectance
g = gdal.Open(os.path.join(inPath, inRaster), gdal.GA_ReadOnly)
if g is None:
print('Can\'t open Analystic MS File. Please Check it.')
sys.exit(1)
# print (g)
geoT = g.GetGeoTransform()
Proj = g.GetProjection()
x_size = g.RasterXSize # Raster xsize
y_size = g.RasterYSize # Raster ysize
bandCount = g.RasterCount
nodata = g.GetRasterBand(1).GetNoDataValue()
toaArray = sp.empty((y_size, x_size, bandCount), dtype=get_datatype(g))
for j in range(bandCount):
dnArray = g.GetRasterBand(j + 1).ReadAsArray()
toaArray[:, :, j] = sp.round_(dnArray * dicMetadata[j + 1] * 10000)
# Write TOA Reflectance Image in GeoTiff
outName = inRaster[:-4] + '_TOA.tif'
# print (outName)
outFolder = os.path.join(
outPath,
inRaster.split('_')[0][:4] + '_' + inRaster.split('_')[0][4:6] +
'_' + inRaster.split('_')[0][6:8])
if not os.path.isdir(outFolder):
os.makedirs(outFolder)
outFile = os.path.join(outFolder, outName)
driver = gdal.GetDriverByName('GTiff')
dataType = gdal_array.NumericTypeCodeToGDALTypeCode(toaArray.dtype)
# dataType=gdal.GetDataTypeName(toaArray.DataType)
ds = driver.Create(outFile, x_size, y_size, bandCount, dataType,
['COMPRESS=LZW'])
ds.SetGeoTransform(geoT)
ds.SetProjection(Proj)
for k in range(bandCount):
ds.GetRasterBand(k + 1).WriteArray(toaArray[:, :, k])
if nodata != None:
ds.GetRasterBand(k + 1).SetNoDataValue(nodata)
g = None
ds = None
driver = None
elif sensor == 1: # RapidEye
# Search MS Image and Metadata XML File
lstFile = [file for file in os.listdir(inPath)]
# print (lstFile)
for file in lstFile:
if file.split('_')[-1] == "Analytic.tif":
inRaster = file
elif file.split('_')[-1] == "metadata.json":
inMetadata = file
# print (inRaster,inMetadata)
# Open Raster File and Convert DN to TOA Reflectance
# Exo-Atmospheric Irradiance (EAI) in W/m²/µm for each band successively 1-B 2-G 3-R 4-RE 5-NIR
dicEAI = {1: 1997.8, 2: 1863.5, 3: 1560.4, 4: 1395., 5: 1124.4}
# Solar Zenithal Angle (90°-sun elevation) Sun elevation is found in Metadata json file
with open(os.path.join(inPath, inMetadata)) as json_file:
content = json.load(json_file)
SOLAR_ZENITH = 90 - float(content['properties']['sun_elevation'])
ACQUISITION_DATE = content['properties']['acquired'][:10]
# print (SOLAR_ZENITH)
# print (ACQUISITION_DATE)
# Compute Earth-Sun Distance at the day of acquisition in Astronomical Units.
# At this time, the Earth-Sun Distance is done on this website : http://www.instesre.org/Aerosols/sol_calc.htm
# lat = 14,64 & lon = -16,44
dicESUN = {
"2017-07-27": 1.015107862579904,
"2017-10-06": 0.99966670968068246,
"2017-10-26": 0.994036745275356,
"2017-11-09": 0.990452817341342,
"2018-03-15": 0.9917799308371985
}
ESUN = float(dicESUN[ACQUISITION_DATE])
# print (ESUN)
# Create Gdal Dataset to compute TOA Reflectance
g = gdal.Open(os.path.join(inPath, inRaster), gdal.GA_ReadOnly)
if g is None:
print('Can\'t open Analystic File. Please Check it.')
sys.exit(1)
# print (g)
geoT = g.GetGeoTransform()
Proj = g.GetProjection()
x_size = g.RasterXSize # Raster xsize
y_size = g.RasterYSize # Raster ysize
bandCount = g.RasterCount
nodata = g.GetRasterBand(1).GetNoDataValue()
toaArray = sp.empty((y_size, x_size, bandCount), dtype=get_datatype(g))
for j in range(bandCount):
dnArray = g.GetRasterBand(j + 1).ReadAsArray()
toaArray[:, :, j] = sp.round_(
dnArray * 0.01 *
(pi *
(ESUN**ESUN) / dicEAI[j + 1] * cos(SOLAR_ZENITH * pi / 180)) *
10000)
# Save TOA Reflectance Image in GeoTiff
outName = inRaster[:-4] + '_TOA.tif'
# print (outName)
outFolder = os.path.join(
outPath,
os.path.basename(inPath).split('_')[0][:4] + '_' +
os.path.basename(inPath).split('_')[0][4:6] + '_' +
os.path.basename(inPath).split('_')[0][6:8])
if not os.path.isdir(outFolder):
os.makedirs(outFolder)
outFile = os.path.join(outFolder, outName)
driver = gdal.GetDriverByName('GTiff')
dataType = gdal_array.NumericTypeCodeToGDALTypeCode(toaArray.dtype)
# dataType=gdal.GetDataTypeName(toaArray.DataType)
ds = driver.Create(outFile, x_size, y_size, bandCount, dataType,
['COMPRESS=LZW'])
ds.SetGeoTransform(geoT)
ds.SetProjection(Proj)
for k in range(bandCount):
ds.GetRasterBand(k + 1).WriteArray(toaArray[:, :, k])
if nodata != None:
ds.GetRasterBand(k + 1).SetNoDataValue(nodata)
g = None
ds = None
driver = None
def trainModel(self,
do_pca=False,
out_dir='./cache',
rftop=40,
class_labels=SP.array(['G1', 'S', 'G2M']),
cv=10,
npc=3,
is_SVM=1,
is_RFE=0,
scale=False):
if not os.path.exists(out_dir):
os.makedirs(out_dir)
CFG = {}
CFG['is_RFE'] = is_RFE # use recursive feature selection (can be slow for large datasets)
CFG['is_SVM'] = is_SVM # use SVM with univariate feature selection (faster than RFE)
CFG['CV_inner'] = cv #inner CV for RFE_CV: either an int or 'LOOCV'
CFG['out_dir'] = out_dir
CFG['do_pca'] = do_pca
CFG['lassotop'] = 20
self.cv = cv
Y = self.Y
labels = self.labels
var_names = self.geneNames
numClasses = self.numClasses
predRF = SP.zeros((len(labels), numClasses))
predSVM = SP.zeros((len(labels), numClasses))
predSVMrbf = SP.zeros((len(labels), numClasses))
predGNB = SP.zeros((len(labels), numClasses))
predLR = SP.zeros((len(labels), numClasses))
predLRall = SP.zeros((len(labels), numClasses))
names_dict = {}
if self.cv == 'LOOCV':
loo = LeaveOneOut(len(labels))
CV_list = (list(iter(loo)))
CV_list.append((SP.array(range(Y.shape[0])),
SP.array(range(Y.shape[0])))) #all data...
else:
skf = StratifiedKFold(labels, n_folds=self.cv)
CV_list = (list(iter(skf)))
CV_list.append((SP.array(range(Y.shape[0])),
SP.array(range(Y.shape[0])))) #all data...
lambda_best = SP.zeros((1, len(CV_list))).ravel()
print("Performing cross validation ...")
for i in range(len(CV_list)):
if i < len(CV_list) - 1:
print("Fold " + str(i + 1) + " of " + str(len(CV_list) - 1))
else:
print("Final model")
# string label for this fold
#get data of a CV run
cv_tr = CV_list[i][0]
cv_tst = CV_list[i][1]
lab_tr = labels[cv_tr]
Ytr = Y[cv_tr, :]
Ytst = Y[cv_tst, :]
lab_tst = labels[cv_tst]
if (i == len(CV_list) - 1):
foldlabel = 'full'
if (self.Y_tst == None):
Ytst = Y[cv_tst, :]
lab_tst = labels[cv_tst]
else:
foldlabel = 'Test'
Ytst = self.Y_tst
lab_tst = self.labels_tst
else:
foldlabel = str(i)
if do_pca >= 1:
npc = npc #3
#do PCA to get features
pcaCC = PCA(n_components=npc, whiten=False)
pcaCC.fit(Ytr)
pcaTst = pcaCC.transform(Ytst)
pcaTr = pcaCC.transform(Ytr)
#selection = SelectKBest(k=1)
#combined_features = FeatureUnion([("pca", pcaCC), ("univ_select", selection)])
combined_features = FeatureUnion([("pca", pcaCC)])
gnb = GaussianNB()
y_pred = gnb.fit(pcaTr, lab_tr).predict_proba(pcaTst)
if i < len(CV_list) - 1:
predGNB[cv_tst, :] = y_pred #[:,1]
else:
predGNB_ts = y_pred #[:,1]
if do_pca == 2:
Ytr = SP.concatenate((Ytr, pcaTr), 1)
Ytst = SP.concatenate((Ytst, pcaTst), 1)
pcnames = []
for pci in range(npc):
pcnames.append('PC' + str(pci + 1))
var_names = SP.concatenate((var_names, SP.array(pcnames)), 1)
print(" Computing random forest ...")
if CFG['is_RFE'] == 1: #Recursive feature selection with SVM
print(" Computing RFE with SVM ...")
svc = SVC(kernel="linear",
probability=False,
class_weight='auto') #use linear SVM for selection
rfecv = RFECV(estimator=svc, step=1, scoring='f1')
param_grid = dict(estimator__C=[0.1, 1, 10, 100, 1000])
clf_rfe = GridSearchCV(
rfecv, param_grid=param_grid, cv=3,
scoring='f1') #GridSearch to find optimal parameters
clf_rfe.fit(Ytr, lab_tr)
svc = SVC(kernel="linear",
probability=False,
C=clf_rfe.best_estimator_.estimator.C,
class_weight='auto') #use linear SVM for selection
if CFG['CV_inner'] == '':
rfecv = RFECV(estimator=svc, step=1, scoring='f1')
elif CFG['CV_inner'] == 'LOOCV':
rfecv = RFECV(estimator=svc,
step=1,
scoring='f1',
cv=LeaveOneOut(len(lab_tr)))
else:
rfecv = RFECV(estimator=svc,
step=1,
scoring='f1',
cv=StratifiedKFold(lab_tr,
n_folds=CFG['CV_inner']))
clf_rfe.best_estimator_.fit(Ytr, lab_tr)
predicted = clf_rfe.best_estimator_.predict(Ytst)
if i < len(CV_list) - 1:
predSVM[cv_tst, :] = predicted
else:
predSVM_ts[cv_tst] = predicted
classifier = svm.SVC(
kernel='rbf',
gamma=0.05,
class_weight='auto',
probability=True) #rbf kernel for prediction
param_grid = dict(C=[0.1, 1], gamma=[1e-1, 1e-2, 1e-3])
clf_rbf = GridSearchCV(classifier,
param_grid=param_grid,
cv=3,
scoring='f1')
clf_rbf.fit(Ytr[:, clf_rfe.best_estimator_.ranking_ == 1],
lab_tr)
clf_rbf.best_estimator_.fit(
Ytr[:, clf_rfe.best_estimator_.ranking_ == 1], lab_tr)
predicted = clf_rbf.best_estimator_.predict_proba(
Ytst[:, clf_rfe.best_estimator_.ranking_ == 1])
if i < len(CV_list) - 1:
predSVMrbf[cv_tst, :] = predicted
fpr, tpr, thresholds = metrics.roc_curve(
lab_tst, predicted[:, 1])
if (i == len(CV_list) - 1) | CFG["CV_plots"] > 0:
PL.figure()
PL.plot(fpr, tpr)
PL.savefig(CFG['out_dir'] + '/RF_SVM_' + foldlabel +
'.pdf')
names_dict[foldlabel + '_SVM'] = self.geneNames[
clf_rfe.best_estimator_.ranking_ == 1]
elif CFG[
'is_SVM'] == 1: #univariate FS with rbf SVM; choose this if you hava a large data set (many features, eg RNAseq)
print(" SVM feature selection ...")
classifier = svm.SVC(kernel='rbf',
gamma=0.05,
class_weight='auto',
probability=True)
selection = SelectKBest(k=1)
combined_features = FeatureUnion([("univ_select", selection)])
X_features = combined_features.fit(Ytr, lab_tr).transform(Ytr)
scaler = preprocessing.StandardScaler().fit(Ytr)
YtrS = scaler.transform(Ytr)
YtstS = scaler.transform(Ytst)
classifier.fit(X_features, lab_tr)
pipeline = Pipeline([("features", combined_features),
("svm", classifier)])
if CFG['do_pca'] == 3:
param_grid = dict(
features__pca__n_components=SP.unique(
SP.round_(
SP.logspace(1.0,
max(SP.log2(Ytr.shape[1]),
SP.log2(10)),
num=min(5, Ytr.shape[1]),
base=2.0))),
features__univ_select__k=SP.unique(
SP.round_(
SP.logspace(3.0,
SP.log2(Ytr.shape[1]),
num=min(10, Ytr.shape[1]),
base=2.0))),
svm__C=[0.1, 1, 10],
svm__gamma=[1e-1, 1e-2, 1e-3])
else:
C_range = 10.**SP.arange(0, 2)
gamma_range = 10.**SP.arange(-5, 1)
param_grid = dict(features__univ_select__k=SP.unique(
SP.round_(
SP.logspace(3.0,
SP.log2(Ytr.shape[1]),
num=min(10, Ytr.shape[1]),
base=2.0))),
svm__C=C_range,
svm__gamma=gamma_range)
clf = GridSearchCV(pipeline,
param_grid=param_grid,
cv=5,
scoring='f1')
clf.fit(YtrS, lab_tr)
print("The best classifier is: ", clf.best_estimator_)
select_best = clf.best_estimator_.get_params(
)['features__univ_select']
#names_dict[foldlabel+'_SVM']=self.geneNames[SP.argsort(-1.0*select_best.scores_)[0:(select_best.k-1)]]
expected = lab_tst
predicted = clf.best_estimator_.predict_proba(YtstS)
if i < len(CV_list) - 1:
predSVM[cv_tst, :] = predicted
else:
predSVM_ts = predicted
#print(clf.best_estimator_)
classifier = svm.SVC(
kernel='rbf',
gamma=0.05,
class_weight='auto',
probability=True) #rbf kernel for prediction
param_grid = dict(C=[1, 10], gamma=[1e-1, 1e-2, 1e-3])
clf_rbf = GridSearchCV(classifier,
param_grid=param_grid,
cv=5,
scoring='f1')
clf_rbf.fit(Ytr, lab_tr)
clf_rbf.best_estimator_.fit(Ytr, lab_tr)
predicted = clf_rbf.best_estimator_.predict_proba(Ytst)
if i < len(CV_list) - 1:
predSVMrbf[cv_tst, :] = predicted
else:
predSVMrbf_ts = predicted
#do lasso with regularisation path
cs = l1_min_c(Ytr, lab_tr, loss='log') * SP.logspace(0, 3)
print(" Computing regularization path ...")
lasso = linear_model.LogisticRegression(C=cs[0] * 10.0,
penalty='l1',
tol=1e-6)
param_grid = dict(C=cs)
clf_lr = GridSearchCV(lasso,
param_grid=param_grid,
cv=5,
scoring='f1')
clf_lr.fit(Ytr, lab_tr)
clf_lr.best_estimator_.fit(Ytr, lab_tr)
lambda_best[i] = clf_lr.best_params_.get('C')
predicted = clf_lr.best_estimator_.predict_proba(Ytst)
clf = linear_model.LogisticRegression(C=cs[0] * 10.0,
penalty='l1',
tol=1e-6)
coefs_ = []
for c in cs:
clf.set_params(C=c)
clf.fit(Ytr, lab_tr)
coefs_.append(clf.coef_.ravel().copy())
if i < len(CV_list) - 1:
predLR[cv_tst, :] = predicted
else:
predLR_ts = predicted
coefs_ = SP.array(coefs_)
# get ordering by importance (how many times they appear)
order = (coefs_ != 0).sum(axis=0).argsort()
order = order[::-1] # descending
# store this order
featrank_lasso = order
showtop = min(Ytr.shape[1], CFG['lassotop'])
clfAll = linear_model.LogisticRegression(C=1e5,
penalty='l2',
tol=1e-6)
clfAll.fit(Ytr, lab_tr)
predicted = clfAll.predict_proba(Ytst)
if i < len(CV_list) - 1:
predLRall[cv_tst, :] = predicted
else:
predLRall_ts = predicted
forest = ExtraTreesClassifier(n_estimators=500,
random_state=0,
criterion="entropy",
bootstrap=False)
#forest = RandomForestClassifier(n_estimators=500,
# random_state=0, criterion="entropy")
forest.fit(Ytr, lab_tr)
pred = forest.predict_proba(Ytst)
#pdb.set_trace()
if i < len(CV_list) - 1:
predRF[cv_tst, :] = pred #[:,1]
else:
predRF_ts = pred #[:,1]
importances = forest.feature_importances_
std = SP.std(
[tree.feature_importances_ for tree in forest.estimators_],
axis=0)
topfeat = min(Ytr.shape[1], rftop)
indices = SP.argsort(importances)[::-1][0:topfeat]
# store full feature ranking
featrank_rf = SP.argsort(importances)[::-1]
# Plot the feature importances of the forest
if (i == len(CV_list) - 1):
PL.figure()
#PL.title("Feature importances, Fold "+foldddPPlabel+', AUC='+str(SP.round_(metrics.auc(fpr, tpr),3)))
PL.title("Feature importances")
#PL.bar(range(topfeat), importances[indices],color="r", yerr=std[indices], align="center")
PL.bar(range(topfeat),
importances[indices],
color="r",
align="center")
PL.xticks(range(topfeat), indices, rotation=70)
PL.gca().set_xticklabels(var_names[indices])
PL.setp(PL.gca().get_xticklabels(), fontsize=8)
PL.xlim([-1, topfeat])
PL.savefig(out_dir + '/RF_featureimportance_' + foldlabel +
'.pdf')
f2 = open(os.path.join(out_dir, 'classification_reportCV.txt'), 'w')
predRFv = SP.argmax(predRF_ts, axis=1) + 1
predRF_trv = SP.argmax(predRF, axis=1) + 1
self.scores = predRF
self.scores_tst = predRF_ts
self.ranking = var_names[indices]
predLRv = SP.argmax(predLR_ts, axis=1) + 1
predLR_trv = SP.argmax(predLR, axis=1) + 1
self.scoresLR = predLR
self.scoresLR_tst = predLR_ts
predLRallv = SP.argmax(predLRall_ts, axis=1) + 1
predLRall_trv = SP.argmax(predLRall, axis=1) + 1
self.scoresLRall = predLRall
self.scoresLRall_tst = predLRall_ts
if CFG['is_SVM'] == 1:
predSVMv = SP.argmax(predSVM_ts, axis=1) + 1
predSVM_trv = SP.argmax(predSVM, axis=1) + 1
self.scoresSVM = predSVM
self.scoresSVM_tst = predSVM_ts
predSVMrbfv = SP.argmax(predSVMrbf_ts, axis=1) + 1
predSVMrbf_trv = SP.argmax(predSVMrbf, axis=1) + 1
self.scoresSVMrbf = predSVMrbf
self.scoresSVMrbf_tst = predSVMrbf_ts
predGNBv = SP.argmax(predGNB_ts, axis=1) + 1
predGNB_trv = SP.argmax(predGNB, axis=1) + 1
self.scoresGNB = predGNB
self.scoresGNB_tst = predGNB_ts
print("Classification report for classifier %s:\n%s\n" %
('Gaussian Naive Bayes',
metrics.classification_report(self.labels, predGNB_trv)))
print("Classification report for classifier %s:\n%s\n" %
('Random Forest',
metrics.classification_report(self.labels, predRF_trv)))
print("Classification report for classifier %s:\n%s\n" %
('LR', metrics.classification_report(self.labels, predLR_trv)))
print("Classification report for classifier %s:\n%s\n" %
('LRall',
metrics.classification_report(self.labels, predLRall_trv)))
if CFG['is_RFE'] == 1:
print(
"Classification report for classifier %s:\n%s\n" %
('SVM ', metrics.classification_report(labels, predSVM > 0.5)),
file=f2)
elif CFG['is_SVM'] == 1:
print("Classification report for classifier %s:\n%s\n" %
('SVM',
metrics.classification_report(self.labels, predSVM_trv)))
print("Classification report for classifier %s:\n%s\n" %
('SVMrbf',
metrics.classification_report(self.labels, predSVMrbf_trv)))
f2.close()
prec = {'S0':[],'S1':[]}
rec = {'S0':[],'S1':[]}
scores = []
min_score = 1
for train, test in cv:
x_train, y_train = x.iloc[train,:], y.iloc[train]
x_test, y_test = x.iloc[test,:], y.iloc[test]
## x_train, y_train = x,y
## x_test, y_test = xt,yt
clf.fit(x_train, y_train)
y_p=clf.predict(x_test)
females=sp.round_(x_train.SexN,2)==-1.36
xf = x_train.loc[females,:].drop(['SexN','SexClass','SexParch','Mr','Master','Rev'],1)
yf = y_train.loc[females]
clf.fit(xf, yf)
yf_p =clf.predict(x_test.drop(['SexN','SexClass','SexParch','Mr','Master','Rev'],1))
males=sp.round_(x_train.SexN,2)==0.74
xm = x_train.loc[males,:].drop(['SexN','SexClass','SexParch','Miss','Mrs'],1)
ym = y_train.loc[males]
clf.fit(xm, ym)
ym_p =clf.predict(x_test.drop(['SexN','SexClass','SexParch','Miss','Mrs'],1))
for i in range(0,len(x_test)):
if (x_test.iloc[i,:].SexN == 1) & (y_p[i] == 0):
y_p[i]=yf_p[i]
@author: james
"""
import matplotlib.pyplot as plt
import scipy
import scipy.stats
df = pd.read_excel(
r"C:\Users\james\Documents\School\Machine Learning & Data mining\Project1\Data\hprice2.xls",
header=None)
cols = range(0, 11)
raw_data = df.get_values()
X = raw_data[:, cols]
size = 30000
x = scipy.arange(size)
y = scipy.int_(scipy.round_(scipy.stats.vonmises.rvs(5, size=size) * 47))
h = plt.hist(y, bins=range(48))
dist_names = ['gamma', 'beta', 'rayleigh', 'norm', 'pareto']
for dist_name in dist_names:
dist = getattr(scipy.stats, dist_name)
param = dist.fit(y)
pdf_fitted = dist.pdf(x, *param[:-2], loc=param[-2],
scale=param[-1]) * size
plt.plot(pdf_fitted, label=dist_name)
plt.xlim(0, 47)
plt.legend(loc='upper right')
plt.show()
# from https://stackoverflow.com/questions/6620471/fitting-empirical-distribution-to-theoretical-ones-with-scipy-python
# Saullo's answer
import matplotlib.pyplot as plt
import scipy
import scipy.stats
size = 20000
x = scipy.arange(size)
# creating the dummy sample (using beta distribution)
y = scipy.int_(scipy.round_(scipy.stats.beta.rvs(6,2,size=size)*47))
# creating the histogram
h = plt.hist(y, bins=range(48), density=True)
dist_names = ['alpha', 'beta', 'arcsine',
'weibull_min', 'weibull_max', 'rayleigh']
for dist_name in dist_names:
dist = getattr(scipy.stats, dist_name)
param = dist.fit(y)
pdf_fitted = dist.pdf(x, *param[:-2], loc=param[-2], scale=param[-1])
plt.plot(pdf_fitted, label=dist_name)
plt.xlim(0,47)
plt.legend(loc='upper left')
plt.show()
#NB : from scipy.stats._continuous_distns import _distn_names #all dist names
#fit() method mentioned by @Saullo Castro provides maximum likelihood estimates (MLE). The best distribution for your data is the one give you the highest can be determined by several different ways: such as
#1, the one that gives you the highest log likelihood.
#2, the one that gives you the smallest AIC, BIC or BICc values (see wiki: http://en.wikipedia.org/wiki/Akaike_information_criterion, basically can be viewed as log likelihood adjusted for number of parameters, as distribution with more parameters are expected to fit better)
#3, the one that maximize the Bayesian posterior probability. (see wiki: http://en.wikipedia.org/wiki/Posterior_probability)
def gprint(G, mtype="obs", bend=5, curve=5, R=1, layout=None, scale=5):
"""
Prints out an automatically layout compressed dbn repesentation of
the graph in TikZ/Latex format
"""
output = StringIO.StringIO()
BE = set()
n = len(G)
if not layout:
g = dict2graph(ecj.cloneBfree(G))
layout = g.layout_fruchterman_reingold(maxiter=50000, coolexp=1.1)
#layout = g.layout_graphopt(niter=50000, node_charge=0.08)
layout.center([0, 0])
layout.scale(float(1 / scipy.absolute(layout.coords).max()))
layout.scale(R)
cc = scipy.round_(array(layout.coords), decimals=4)
else:
g = dict2graph(ecj.cloneBfree(G))
cc = array(layout.coords)
paintSCC(g, colors)
for i in range(0, n):
node = g.vs[i]['label']
rc = g.vs[i]["color"]
print >>output, "{ \\definecolor{mycolor}{RGB}{"\
+str(rc[0])+","+str(rc[1])+","+str(rc[2])+"}"
mcolor = "fill = {rgb: red,"+str(rc[0])+"; green,"+str(rc[1])+\
"; blue,"+str(rc[2])+"}"
print >>output, "\\node["+mtype+", fill=mycolor] ("+node+") at ("+\
str(cc[i][0])+","+str(cc[i][1])+") {"+node+"};}"
for i in range(0, n):
v = g.vs[i]['label']
ll = [v + '/' + u for u in G[v]]
for l in ll:
a, b = l.split('/')
if G[a][b].intersection([(edge_type['bidirected'], 0)]):
if not (BE.intersection([(a, b)])
or BE.intersection([(b, a)])):
print >>output,' \\draw[pilip, on layer=back] ('+\
a+') -- ('+b+');'
if G[a][b].intersection([(edge_type['directed'], 1)]):
if a == b:
dff = cc[g.vs['label'].index(a)] - scipy.mean(cc, 0)
ang = scipy.arctan2(dff[1], dff[0])
ang_a = scipy.rad2deg(ang)
print >>output,"\\path[overlay,draw,pil] ("+a+")" +\
" .. controls +("+ "%.5f" % (bend+ang_a) +\
":"+ fstr % (2*curve)+"mm) and +("+\
"%.5f" % (ang_a-bend)+\
":"+"%.5f" % (2*curve)+"mm) .. ("+b+");"
else:
dff = cc[g.vs['label'].index(b)] \
- cc[g.vs['label'].index(a)]
ang = scipy.arctan2(dff[1], dff[0])
ang_a = scipy.rad2deg(ang)
ang_b = ang_a + 180
print >>output,"\\path[overlay,draw,pil] ("+a+")" +\
" .. controls +("+\
"%.5f" % (bend+ang_a) +\
":"+fstr % (curve)+"mm) and +("+\
fstr % (ang_b-bend)+\
":"+fstr % (curve)+"mm) .. ("+b+");"
return output
plt.figure(2)
ax = plt.axes(projection='3d')
ax.plot3D(x_o/1000,y_o/1000,z_o/1000)
plt.savefig('3d.png')
''' DERIVA ODEINT '''
deriva = sp.zeros(len(tiempo))
for i in range(len(tiempo)):
deriva[i] = sp.sqrt(sp.dot((z_odeint[i,:3] - [x[i],y[i],z[i]]), (z_odeint[i,:3] - [x[i],y[i],z[i]])))
z_f = sp.sqrt(sp.dot([x[i],y[i],z[i]],[x[i],y[i],z[i]]))
error = sp.round_(deriva[-1]/z_f,2)
print (f'Error = {error*100} %')
dist_max = sp.round_(sp.amax(deriva)/1000,1)
plt.figure(3)
plt.title(f'Distancia entre eulerint y odeint, $δ_m$ = {dist_max} (km)')
plt.ylabel('Deriva, δ [KM]')
plt.xlabel('Tiempo, $t$ [h]')
plt.plot(t/3600,deriva/1000)
plt.tight_layout()
plt.savefig('deriva_odeint.png')
def plotPerformance(self,
perfType='ROC',
plot_test=True,
out_dir='./cache',
class_labels=['G1 phase', 'S phase', 'G2M phase'],
method='RF'):
plparams = {
'backend': 'pdf',
'axes.labelsize': 14,
'text.fontsize': 14,
'legend.fontsize': 13,
'xtick.labelsize': 14,
'ytick.labelsize': 14,
'text.usetex': False
}
PL.rcParams.update(plparams)
assert (perfType in ['ROC', 'PR'])
if plot_test == True:
if method == 'RF':
labs = self.labels_tst
scores = self.scores_tst
elif method == 'LR':
labs = self.labels_tst
scores = self.scoresLR_tst
elif method == 'LRall':
labs = self.labels_tst
scores = self.scoresLRall_tst
elif method == 'GNB':
labs = self.labels_tst
scores = self.scoresGNB_tst
elif method == 'SVM':
labs = self.labels_tst
scores = self.scoresSVM_tst
elif method == 'SVMrbf':
labs = self.labels_tst
scores = self.scoresSVMrbf_tst
else:
if method == 'RF':
labs = self.labels
scores = self.scores
elif method == 'LR':
labs = self.labels
scores = self.scoresLR
elif method == 'LRall':
labs = self.labels
scores = self.scoresLRall
elif method == 'GNB':
labs = self.labels
scores = self.scoresGNB
elif method == 'SVM':
labs = self.labels
scores = self.scoresSVM
elif method == 'SVMrbf':
labs = self.labels
scores = self.scoresSVMrbf
PL.figure()
#col = ['r', 'b', 'g']
col = ['#1b9e77', '#d95f02', '#7570b3']
aucList = SP.zeros((scores.shape[1], ))
for ind in range(scores.shape[1]):
labels_i = labs.copy()
scores_i = scores[:, ind]
labels_i[labs == ind + 1] = 1
labels_i[labs != ind + 1] = 0
if perfType == 'ROC':
fpr, tpr, thresholds = metrics.roc_curve(labels_i, scores_i)
aucList[ind] = metrics.auc(fpr, tpr)
elif perfType == 'PR':
fpr, tpr, thresholds = metrics.precision_recall_curve(
labels_i, scores_i)
PL.plot(fpr, tpr, '-', c=col[ind])
if perfType == 'ROC':
PL.title("ROC curve ccClassify")
PL.xlabel('FPR')
PL.ylabel('TPR')
leg_str = list()
for i in range(len(class_labels)):
leg_str.append(class_labels[i] + ', AUC = ' +
str(SP.round_(aucList[i], 3)))
PL.legend(leg_str, loc='lower-right')
ax = plt.gca()
ax.spines["right"].set_visible(False)
ax.spines["top"].set_visible(False)
ax.get_xaxis().tick_bottom()
ax.get_yaxis().tick_left()
PL.savefig(out_dir + '/ROC_test' + str(plot_test) + '_' + method +
'.pdf',
bbox_inches='tight')
else:
PL.title("PR curve ccClassify")
PL.xlabel('Precision')
PL.ylabel('Recall')
leg_str = list()
for i in range(len(class_labels)):
leg_str.append(class_labels[i]
) #+', AUC = '+str(SP.round_(aucList[i],3)))
PL.legend(leg_str, loc='lower-right')
ax = plt.gca()
ax.spines["right"].set_visible(False)
ax.spines["top"].set_visible(False)
ax.get_xaxis().tick_bottom()
ax.get_yaxis().tick_left()
PL.savefig(out_dir + '/ROC_test' + str(plot_test) + '_' + method +
'.pdf',
bbox_inches='tight')
def prices(request):
try:
required_options = set([o for o in request.POST.keys() if request.POST[o] == u'true'])
published_options = sorted(
#PublishOptionContract.objects.filter(optiondefinition_id=option.id),
PublishOptionContract.objects.all(),
key = lambda x : x.strike)
published_option_definitions = set([])
for o in published_options:
published_option_definitions.add(o.optiondefinition)
futures = sorted(
Future.objects.all() ,
key = lambda x : x.name)
futures = [el for el in futures
if el.expiry_date > datetime.datetime.today()]
option_definitions = OptionDefinition.objects.all()
#if len(required_options) == 0:
# required_options = set(option_definitions)
option_definitions = [el for el in option_definitions
if el.expiry_date > datetime.datetime.today() and el.name in required_options]
#option_definition_dict = dict([(o, o.name) for o in option_definitions])
futuredict = collections.OrderedDict()
for future in futures:
futuredict[future] = collections.OrderedDict()
for option_def in option_definitions:
if option_def.future == future:
#futuredict[future][option_definition_dict[option_def]] = collections.OrderedDict()
futuredict[future][option_def] = collections.OrderedDict()
publishedcontracts = sorted(
[i for i in published_options if i.optiondefinition == option_def],
key = lambda x : x.strike)
if len(publishedcontracts) > 0:
futuredict[future][option_def]['published'] = True
#futuredict[future][option_definition_dict[option_def]]['published_time'] = publishedcontracts[0].publish_time
futuredict[future][option_def]['published_time'] = publishedcontracts[0].publish_time
table_data = []
atm_strike = hf.ATM_strike([c.strike for c in publishedcontracts], publishedcontracts[0].future_value)
time2expiry = hf.diff_dates_year_fraction(option_def.expiry_date, publishedcontracts[0].publish_time)
random_column = []
for pos, o in enumerate(publishedcontracts):
if o.strike == atm_strike:
random_column.append(scipy.round_(hf.black_pricer(time2expiry, o.future_value, o.strike, o.vol, call = True) + hf.black_pricer(time2expiry, o.future_value, o.strike, o.vol, call = False), decimals = 5))
else:
random_column.append('')
published_strikes = []
published_vols = []
for pos, c in enumerate(publishedcontracts):
table_data.append(
(
scipy.round_(c.strike, decimals=5),
scipy.round_(c.call_value, decimals=5),
scipy.round_(c.put_value, decimals=5),
random_column[pos],
scipy.round_(c.call_delta, decimals=2),
scipy.round_(c.put_delta, decimals=2),
scipy.round_(c.gamma, decimals=3),
scipy.round_(c.theta, decimals=2),
scipy.round_(c.vega, decimals=2),
scipy.round_(c.vol, decimals=5),
scipy.round_(c.change, decimals=5),
)
)
published_strikes.append(c.strike)
published_vols.append(c.vol)
#futuredict[future][option_definition_dict[option_def]]['table_data'] = table_data
futuredict[future][option_def]['table_data'] = table_data
optioncontracts = sorted(
OptionContract.objects.filter(optiondefinition_id=option_def.id),
key = lambda x : x.strike)
strikes = [o.strike for o in optioncontracts]
all_strikes = set(strikes)
all_strikes.update(set(published_strikes))
all_strikes = sorted(list(all_strikes))
published_vols = hf.remap_values(published_vols, published_strikes, all_strikes)
futuredict[future][option_def]['published_vols'] = published_vols
bids = hf.remap_values([o.bid if o.bid > 0 else None for o in optioncontracts], strikes, all_strikes)
asks = hf.remap_values([o.ask if o.ask > 0 else None for o in optioncontracts], strikes, all_strikes)
last_trade_value = hf.remap_values([o.last_trade_value if o.last_trade_value > 0 else None for o in optioncontracts], strikes, all_strikes)
values = hf.remap_values([o.value if o.value > 0 else None for o in optioncontracts], strikes, all_strikes)
future = option_def.future
callbool = [True if o.easy_screen_mnemonic[-3] == 'c' else False for o in optioncontracts]
today = datetime.date.today()
time2expiry = hf.diff_dates_year_fraction(option_def.expiry_date, today)
bid_vols = [hf.black_pricer_vol(time2expiry, future.bid, K, Val, call) for K, Val, call in zip(strikes, bids, callbool)]
ask_vols = [hf.black_pricer_vol(time2expiry, future.bid, K, Val, call) for K, Val, call in zip(strikes, asks, callbool)]
last_trade_vols = [hf.black_pricer_vol(time2expiry, future.last_trade_value, K, Val, call) for K, Val, call in zip(strikes, last_trade_value, callbool)]
value_vols = [hf.black_pricer_vol(time2expiry, future.bid, K, Val, call) for K, Val, call in zip(strikes, values, callbool)]
value_vols = hf.spline_interpolate(value_vols, strikes)
futuredict[future][option_def]['strikes'] = json.dumps(all_strikes)
futuredict[future][option_def]['bid_vols'] = json.dumps(bid_vols)
futuredict[future][option_def]['ask_vols'] = json.dumps(ask_vols)
futuredict[future][option_def]['last_trade_vols'] = json.dumps(last_trade_vols)
futuredict[future][option_def]['value_vols'] = json.dumps(value_vols)
else:
futuredict[future][option_def]['published'] = False
if len(required_options) > 0:
containsdata = True
else:
containsdata = False
template = loader.get_template('prices.html')
context = RequestContext(request, {
'futuredict': futuredict,
'containsdata' : containsdata
})
return HttpResponse(template.render(context))
except Exception as e:
#print e
raise Http404
return render(request, 'prices.html')
salmapFile = matpyfiles['salmapFile'][0][0][0]
iou = params['params'][0]['windows'][0]['iou'][0][0][0][0]
scales = params['params'][0]['windows'][0]['scale'][0][0][0]
aspRatio = params['params'][0]['windows'][0]['aspRatio'][0][0][0]
# Initializing and loading files
salmap = sio.loadmat(salmapFile)
salmap = salmap['salmap']
windows = []
rows, cols = salmap.shape[0], salmap.shape[1]
imgArea = rows * cols
stepFactor = ((1 - iou) / (1 + iou))
# Generate Windows
for scale in scales:
winArea = scale * imgArea
for ar in aspRatio:
width = int(sp.round_(sqrt(winArea * ar)))
height = int(sp.round_(sqrt(float(winArea) / ar)))
stepH = int(sp.round_(width * stepFactor))
stepV = int(sp.round_(height * stepFactor))
for rmin in xrange(0, rows - height + 1, stepV):
for cmin in xrange(0, cols - width + 1, stepH):
rmax = min(rows - 1, rmin + height)
cmax = min(cols - 1, cmin + width)
windows.append([cmin, rmin, cmax, rmax])
windows = sp.array(windows) + 1
sio.savemat(windowsFile, mdict={'windows': windows})
wgm_phi = sp.cos(m * phi) + sp.sin(m * phi) * sp.sqrt(-1)
heavymap = sp.heaviside((r / a) - 1, 1)
if (m == 7):
wgm = sp.real(wgm_phi * int_wgm_rho * (1 - heavymap) +
B * wgm_phi * ext_wgm_rho * heavymap)
else:
wgm = wgm + sp.real(wgm_phi * int_wgm_rho * (1 - heavymap) +
B * wgm_phi * ext_wgm_rho * heavymap)
m += 1
#%%draw on circle
heavyp = sp.heaviside(r - 0.995 * a, 1)
heavyn = sp.heaviside(1.005 * a - r, 1)
xlab = sp.round_(x, 2)
ylab = sp.round_(y, 2)
circle = pd.DataFrame(heavyp * heavyn, index=ylab, columns=xlab)
#%% make heatmap of function
g = sns.heatmap(wgm,
cmap="RdBu_r",
xticklabels=True,
yticklabels=True,
square=True)
circle_palette = [(0xFF / 0xFF, 0xFF / 0xFF, 0xFF / 0xFF, 0.001),
(0xD1 / 0xFF, 0xEC / 0xFF, 0x9C / 0xFF, 1)]
cmap = mpl.colors.ListedColormap(circle_palette)
c = sns.heatmap(circle,
cmap=cmap,
xticklabels=100,
def gprint(G, mtype="obs", bend=5, curve=5, R=1, layout=None, scale=5):
"""
Prints out an automatically layout compressed dbn repesentation of
the graph in TikZ/Latex format
"""
output = StringIO.StringIO()
BE = set()
n = len(G)
if not layout:
g = dict2graph(ecj.cloneBfree(G))
layout = g.layout_fruchterman_reingold(maxiter=50000, coolexp=1.1)
# layout = g.layout_graphopt(niter=50000, node_charge=0.08)
layout.center([0, 0])
layout.scale(float(1 / scipy.absolute(layout.coords).max()))
layout.scale(R)
cc = scipy.round_(array(layout.coords), decimals=4)
else:
g = dict2graph(ecj.cloneBfree(G))
cc = array(layout.coords)
paintSCC(g, colors)
for i in range(0, n):
node = g.vs[i]['label']
rc = g.vs[i]["color"]
print >>output, "{ \\definecolor{mycolor}{RGB}{"\
+ str(rc[0]) + "," + str(rc[1]) + "," + str(rc[2]) + "}"
mcolor = "fill = {rgb: red," + str(rc[0]) + "; green," + str(rc[1]) +\
"; blue," + str(rc[2]) + "}"
print >>output, "\\node[" + mtype + ", fill=mycolor] (" + node + ") at (" +\
str(cc[i][0]) + "," + str(cc[i][1]) + ") {" + node + "};}"
for i in range(0, n):
v = g.vs[i]['label']
ll = [v + '/' + u for u in G[v]]
for l in ll:
a, b = l.split('/')
if G[a][b].intersection([(edge_type['bidirected'], 0)]):
if not(BE.intersection([(a, b)]) or BE.intersection([(b, a)])):
print >>output, ' \\draw[pilip, on layer=back] (' +\
a + ') -- (' + b + ');'
if G[a][b].intersection([(edge_type['directed'], 1)]):
if a == b:
dff = cc[g.vs['label'].index(a)] - scipy.mean(cc, 0)
ang = scipy.arctan2(dff[1], dff[0])
ang_a = scipy.rad2deg(ang)
print >>output, "\\path[overlay,draw,pil] (" + a + ")" +\
" .. controls +(" + "%.5f" % (bend + ang_a) +\
":" + fstr % (2 * curve) + "mm) and +(" +\
"%.5f" % (ang_a - bend) +\
":" + "%.5f" % (2 * curve) + "mm) .. (" + b + ");"
else:
dff = cc[g.vs['label'].index(b)] \
- cc[g.vs['label'].index(a)]
ang = scipy.arctan2(dff[1], dff[0])
ang_a = scipy.rad2deg(ang)
ang_b = ang_a + 180
print >>output, "\\path[overlay,draw,pil] (" + a + ")" +\
" .. controls +(" +\
"%.5f" % (bend + ang_a) +\
":" + fstr % (curve) + "mm) and +(" +\
fstr % (ang_b - bend) +\
":" + fstr % (curve) + "mm) .. (" + b + ");"
return output
print '\nwithout using cache' print tn, '(*n)' print '\n2nd\t4th\t6th\t8th\t(spatial order)' print '1D' for n,nx in [(256,256), (512,512)]: print '(%d=%d)' % (n,nx) n2 = 6*n + 3*n + (2*n+1) + (2*n+1) n4 = 6*n + 7*n + (4*n+3) + (4*n+3) n6 = 6*n + 11*n + (6*n+5) + (6*n+5) n8 = 6*n + 15*n + (8*n+7) + (8*n+7) #rn = 6.*n + (2*p-1)*n + (p*n+p-1) + (p*n+p-1) rn = 6.*n + (4*p-1)*n + 2*(p-1) #print rn ratio = rn/(tn*n)*100 print sc.round_(ratio,2), '%' print '\n2D' for n,nx,ny in [(256,16,16), (512,32,16)]: print '(%d=%dx%d)' % (n,nx,ny) n2 = 6*n + (2*n+nx) + (2*n+ny) + (n+(nx+ny)) n4 = 6*n + (4*n+3*nx) + (4*n+3*ny) + (n+3*(nx+ny)) #rn = 6.*n + (p*n+(p-1)*nx) + (p*n+(p-1)*ny) + (n+(p-1)*(nx+ny)) rn = 6.*n + (2*p+1)*n + 2*(p-1)*(nx+ny) ratio = rn/(tn*n)*100 print sc.round_(ratio,2), '%' print '\n3D' for n,nx,ny,nz in [(256,16,4,4), (512,16,8,4)]: print '(%d=%dx%dx%d)' % (n,nx,ny,nz) n2 = 6*n + 3*n + 2*(nx*ny+ny*nz+nz*nx)
def __pyramid_ski__(frame,steps = 7,scale = 2):
# create image pyramid from img, skimage implemetation
py = pyramid_gaussian(frame, downscale = scale)
return [sp.round_(255.0*e).astype(sp.uint8) for e in py][:steps]
def getCorrection(x,y,x_sv, y_sv):
'''looks up correction in calculated vector fields'''
xi = np.maximum(np.minimum(sp.round_(x/100).astype('i'), x_sv.shape[0]),0)
yi = np.maximum(np.minimum(sp.round_(y/100).astype('i'), x_sv.shape[1]),0)
return (x_sv[xi, yi],y_sv[xi, yi])
