def test_power_spectra(r0, N, delta, L0, l0):
N*= 10
phase_screen = atmosphere.ft_phase_screen(r0, N, delta, L0, l0)
phase_screen = phase_screen[:N/10, :N/10]
power_spec_2d = numpy.fft.fft2(phase_screen, s=(N*2, N*2))
plt.figure()
plt.imshow(numpy.abs(numpy.fft.fftshift(power_spec_2d)), interpolation='nearest')
power_spec = circle.aziAvg(numpy.abs(numpy.fft.fftshift(power_spec_2d)))
power_spec /= power_spec.sum()
freqs = numpy.fft.fftfreq(power_spec_2d.shape[0], delta)
# Theoretical Model of Power Spectrum
print freqs
plt.figure()
plt.plot(freqs[:freqs.size/2], power_spec)
plt.xscale('log')
plt.yscale('log')
plt.show()
return None
def get_OH(OIII4363,OIII4959,OIII5007,Hb):
Te = np.arange(5000,20000,1)
t3 = Te/1e4
ne = 100 # cm^-3
x = 1e-4*ne*t3**(-0.5)
C_T = (8.44-1.09*t3+0.5*t3**2.-0.08*t3**3.)*(1.+0.0004*x)/(1.+0.044*x)
log_OIII_ratio = 1.432/t3+np.log10(C_T)
log_OIII_ratio_obs = np.log10((OIII4959+OIII5007)/OIII4363)
Te_obs = []
plt.clf()
plt.plot(Te,log_OIII_ratio,color='black',marker='.',linestyle='none')
plt.yscale('log')
for i in range(len(OIII4363)):
plt.axhline(log_OIII_ratio_obs[i],linestyle='--')
d_ratio = abs(log_OIII_ratio_obs[i]-log_OIII_ratio)
min_d_ratio = min(d_ratio)
min_sub = list(d_ratio).index(min_d_ratio)
Te_obs.append(Te[min_sub])
plt.xlim(10000,20000)
plt.ylim(1.,3)
plt.xlabel('Te')
plt.ylabel('(OIII4959+5007)/OIII4363')
Te_obs = np.array(Te_obs)
t3_obs = Te_obs/1e4
logOIIIH = np.log10((OIII4959+OIII5007)/Hb)+6.200+1.251+1.251/t3_obs - \
5*np.log10(t3_obs)-0.014*t3_obs
t2_obs = -0.577+t3*(2.065-0.498*t3)
logOIIH = np.log10(OII3727/Hb)+5.961+1.676/t2_obs-0.4*np.log10(t2_obs) - \
0.034*t2_obs+np.log10(1+1.35*x)
OH = 10**(logOIIIH-12.)+10**(logOIIIH-12.)
logOH = 12 + np.log10(OH)
return Te_obs,logOIIH,logOIIIH,logOH
def test_simple_gen(self):
self_con = .8
other_con = 0.05
g = self.gen.gen_stoch_blockmodel(min_degree=1, blocks=5, self_con=self_con, other_con=other_con,
powerlaw_exp=2.1, degree_seq='powerlaw', num_nodes=1000, num_links=3000)
deg_hist = vertex_hist(g, 'total')
res = fit_powerlaw.Fit(g.degree_property_map('total').a, discrete=True)
print 'powerlaw alpha:', res.power_law.alpha
print 'powerlaw xmin:', res.power_law.xmin
if len(deg_hist[0]) != len(deg_hist[1]):
deg_hist[1] = deg_hist[1][:len(deg_hist[0])]
print 'plot degree dist'
plt.plot(deg_hist[1], deg_hist[0])
plt.xscale('log')
plt.xlabel('degree')
plt.ylabel('#nodes')
plt.yscale('log')
plt.savefig('deg_dist_test.png')
plt.close('all')
print 'plot graph'
pos = sfdp_layout(g, groups=g.vp['com'], mu=3)
graph_draw(g, pos=pos, output='graph.png', output_size=(800, 800),
vertex_size=prop_to_size(g.degree_property_map('total'), mi=2, ma=30), vertex_color=[0., 0., 0., 1.],
vertex_fill_color=g.vp['com'],
bg_color=[1., 1., 1., 1.])
plt.close('all')
print 'init:', self_con / (self_con + other_con), other_con / (self_con + other_con)
print 'real:', gt_tools.get_graph_com_connectivity(g, 'com')
def smooth_color_prior(size=64, sigma=5, do_plot=False):
prior_prob = np.load(os.path.join(data_dir, "CelebA_%s_prior_prob.npy" % size))
# add an epsilon to prior prob to avoid 0 vakues and possible NaN
prior_prob += 1E-3 * np.min(prior_prob)
# renormalize
prior_prob = prior_prob / (1.0 * np.sum(prior_prob))
# Smooth with gaussian
f = interp1d(np.arange(prior_prob.shape[0]),prior_prob)
xx = np.linspace(0,prior_prob.shape[0] - 1, 1000)
yy = f(xx)
window = gaussian(2000, sigma) # 2000 pts in the window, sigma=5
smoothed = convolve(yy, window / window.sum(), mode='same')
fout = interp1d(xx,smoothed)
prior_prob_smoothed = np.array([fout(i) for i in range(prior_prob.shape[0])])
prior_prob_smoothed = prior_prob_smoothed / np.sum(prior_prob_smoothed)
# Save
file_name = os.path.join(data_dir, "CelebA_%s_prior_prob_smoothed.npy" % size)
np.save(file_name, prior_prob_smoothed)
if do_plot:
plt.plot(prior_prob)
plt.plot(prior_prob_smoothed, "g--")
plt.plot(xx, smoothed, "r-")
plt.yscale("log")
plt.show()
def nova_plot():
erg2mev=624151.
fig=plot.figure()
yrange = [1e-6,2e-4]
xrange = [1e-1,1e5]
plot.fill_between([0.2,10e3],[yrange[1],yrange[1]],[yrange[0],yrange[0]],facecolor='yellow',interpolate=True,color='yellow',alpha=0.5)
plot.annotate('AMEGO',xy=(3,9e-5),xycoords='data',fontsize=26,color='black')
lat=ascii.read("data/NMon2012.LAT.dat",names=['energy','en_low','en_high','flux','flux_err','tmp'])
plot.scatter(lat['energy'],lat['flux']*erg2mev,color='red')
plot.errorbar(lat['energy'],lat['flux']*erg2mev,xerr=[lat['en_low'],lat['en_high']],yerr=lat['flux_err']*erg2mev,ecolor='red',capsize=0,fmt='none')
latul=ascii.read("data/NMon2012.LAT.limits.dat",names=['energy','en_low','en_high','flux','tmp1','tmp2','tmp3','tmp4'])
plot.errorbar(latul['energy'],latul['flux']*erg2mev,xerr=[latul['en_low'],latul['en_high']],yerr=0.5*latul['flux']*erg2mev,uplims=True,ecolor='red',capsize=0,fmt='none')
plot.scatter(latul['energy'],latul['flux']*erg2mev,color='red')
leptonic=ascii.read("data/sp-NMon12-IC-best-fit-1MeV-30GeV.txt",names=['energy','flux'],data_start=1)
hadronic=ascii.read("data/sp-NMon12-pi0-and-secondaries.txt",names=['energy','flux1','flux2'],data_start=1)
plot.plot(leptonic['energy'],leptonic['flux']*erg2mev,'r--',color='black',lw=2,label='Leptonic')
plot.plot(hadronic['energy'],hadronic['flux2']*erg2mev,color='black',lw=2,label='Hadronic+Secondary Leptons')
plot.legend(loc='upper right',fontsize='small',frameon=False,framealpha=0.5)
plot.xscale('log')
plot.yscale('log')
plot.ylim(yrange)
plot.xlim(xrange)
plot.xlabel(r'Energy (MeV)')
plot.ylabel(r'Energy$^2 \times $ Flux (Energy) (erg cm$^{-2}$ s$^{-1}$)')
plot.title('Nova V339 Del 2013')
plot.savefig('Nova_SED.png', bbox_inches='tight')
plot.savefig('Nova_SED.eps', bbox_inches='tight')
plot.show()
plot.close()
def plotFeaturePDF(ift, pft, outbase, fmin=0.0, fmax=1.0, fstep=0.01):
"""
Plot a comparison between the input feature distribution and the
feature distribution of the predicted halos
"""
plt.clf()
nfbins = ( fmax - fmin ) / fstep
fbins = np.logspace( fmin, fmax, nfbins )
fcen = ( fbins[:-1] + fbins[1:] ) / 2
plt.xscale( 'log', nonposx='clip' )
plt.yscale( 'log', nonposy='clip' )
ic, e, p = plt.hist( ift, fbins, label='Original Halos', alpha=0.5, normed=True )
pc, e, p = plt.hist( pft, fbins, label='Added Halos', alpha=0.5, normed=True )
plt.legend()
plt.xlabel( r'$\delta$' )
plt.savefig( outbase+'_fpdf.png' )
fdtype = np.dtype( [ ('fcen', float), ('ifcounts', float), ('pfcounts', float) ] )
fd = np.ndarray( len(fcen), dtype = fdtype )
fd[ 'mcen' ] = fcen
fd[ 'imcounts' ] = ic
fd[ 'pmcounts' ] = pc
fitsio.write( outbase+'_fpdf.fit', fd )
def plot_ccdf(values, xscale, yscale):
pylab.yscale(yscale)
cdf = Cdf.MakeCdfFromList(values)
values, prob = cdf.Render()
pylab.xscale(xscale)
compProb = [1 - e for e in prob]
pylab.plot(values, compProb)
def compareMassRatioVsRedshift(his, hiserr, mine, myerr, hisfield, hisid, myfield, myid, z):
import matplotlib.pylab as plt
a = []
b = []
c = []
d = []
e = []
z_list = []
for i in range(0, np.shape(mine)[0]):
if mine[i] != 0:
#his_id_idx = np.where(hisid == myid[i])[0]
pos_id_idx = np.where(hisid == myid[i])[0]
if len(pos_id_idx) == 1:
his_id_idx = int(pos_id_idx)
else:
pos_field_idx = np.where(hisfield == myfield[i])[0]
for ii in range(0, len(pos_field_idx)):
for iii in range(0, len(pos_id_idx)):
if pos_field_idx[ii] == pos_id_idx[iii]:
his_id_idx = int(pos_id_idx[iii])
#print str(myid[i]), str(myfield[i]), str(hisid[his_id_idx]), str(hisfield[his_id_idx])
assert myfield[i] == hisfield[his_id_idx]
assert myid[i] == hisid[his_id_idx]
if his[his_id_idx] != -999:
a.append(his[his_id_idx])
b.append(hiserr[his_id_idx])
c.append(mine[i])#-np.log10(((1.0+z[i]))))
d.append(myerr[i,0])#-np.log10(((1.0+z[i]))))
e.append(myerr[i,1])#-np.log10(1.0+z[i]))
z_list.append(z[i])
# since errors are just percentile, need difference from median
d = np.asarray(c)-np.asarray(d)
e = np.asarray(e)-np.asarray(c)
c_a = 10**np.array(c)
a_a = 10**np.array(a)
ratio = c_a/a_a
plt.errorbar(z_list, ratio, fmt='o',
color='b', capsize=0, alpha=0.50)
# plot the y=x line
x = np.linspace(np.min(z_list),
np.max(z_list),
10)
plt.plot(x, np.ones(len(x)), 'k--')
plt.yscale('log')
plt.xlabel("Redshift")
plt.ylabel("MCSED / Alex's [note: ratio of actual masses]")
plt.title("Redshift vs Mass Ratio (MCSED/Alex)")
#plt.legend(['With Neb. Emis.'],loc=0)#, 'W/o Neb. Emis'], loc=0)
plt.show()
def plotMassFunction(im, pm, outbase, mmin=9, mmax=13, mstep=0.05):
"""
Make a comparison plot between the input mass function and the
predicted projected correlation function
"""
plt.clf()
nmbins = ( mmax - mmin ) / mstep
mbins = np.logspace( mmin, mmax, nmbins )
mcen = ( mbins[:-1] + mbins[1:] ) /2
plt.xscale( 'log', nonposx = 'clip' )
plt.yscale( 'log', nonposy = 'clip' )
ic, e, p = plt.hist( im, mbins, label='Original Halos', alpha=0.5, normed = True)
pc, e, p = plt.hist( pm, mbins, label='Added Halos', alpha=0.5, normed = True)
plt.legend()
plt.xlabel( r'$M_{vir}$' )
plt.ylabel( r'$\frac{dN}{dM}$' )
#plt.tight_layout()
plt.savefig( outbase+'_mfcn.png' )
mdtype = np.dtype( [ ('mcen', float), ('imcounts', float), ('pmcounts', float) ] )
mf = np.ndarray( len(mcen), dtype = mdtype )
mf[ 'mcen' ] = mcen
mf[ 'imcounts' ] = ic
mf[ 'pmcounts' ] = pc
fitsio.write( outbase+'_mfcn.fit', mf )
def compute_color_prior(size=64, do_plot=False):
# Load the gamut points location
q_ab = np.load(os.path.join(data_dir, "pts_in_hull.npy"))
if do_plot:
plt.figure(figsize=(15, 15))
gs = gridspec.GridSpec(1, 1)
ax = plt.subplot(gs[0])
for i in range(q_ab.shape[0]):
ax.scatter(q_ab[:, 0], q_ab[:, 1])
ax.annotate(str(i), (q_ab[i, 0], q_ab[i, 1]), fontsize=6)
ax.set_xlim([-110,110])
ax.set_ylim([-110,110])
with h5py.File(os.path.join(data_dir, "CelebA_%s_data.h5" % size), "a") as hf:
# Compute the color prior over a subset of the training set
# Otherwise it is quite long
X_ab = hf["training_lab_data"][:100000][:, 1:, :, :]
npts, c, h, w = X_ab.shape
X_a = np.ravel(X_ab[:, 0, :, :])
X_b = np.ravel(X_ab[:, 1, :, :])
X_ab = np.vstack((X_a, X_b)).T
if do_plot:
plt.hist2d(X_ab[:, 0], X_ab[:, 1], bins=100, norm=LogNorm())
plt.xlim([-110, 110])
plt.ylim([-110, 110])
plt.colorbar()
plt.show()
plt.clf()
plt.close()
# Create nearest neighbord instance with index = q_ab
NN = 1
nearest = nn.NearestNeighbors(n_neighbors=NN, algorithm='ball_tree').fit(q_ab)
# Find index of nearest neighbor for X_ab
dists, ind = nearest.kneighbors(X_ab)
# We now count the number of occurrences of each color
ind = np.ravel(ind)
counts = np.bincount(ind)
idxs = np.nonzero(counts)[0]
prior_prob = np.zeros((q_ab.shape[0]))
for i in range(q_ab.shape[0]):
prior_prob[idxs] = counts[idxs]
# We turn this into a color probability
prior_prob = prior_prob / (1.0 * np.sum(prior_prob))
# Save
np.save(os.path.join(data_dir, "CelebA_%s_prior_prob.npy" % size), prior_prob)
if do_plot:
plt.hist(prior_prob, bins=100)
plt.yscale("log")
plt.show()
def plot_change(alldata,thisdata,name):
pl.style.use('ggplot')
f, ax = pl.subplots()
ax.loglog(alldata,'.',label='All mirNA collected in DB')
ax.loglog(alldata.index(thisdata),thisdata,'o',ms=9,label=name)
ax.legend(loc=2)
pl.ylabel('Expression change')
pl.yscale('log')
pl.ylim(alldata[2],alldata[-2])
pl.yticks(np.logspace(np.log10(alldata[2]),np.log10(alldata[-2]),10),[i.round(2) for i in np.logspace(np.log10(alldata[2]),np.log10(alldata[-2]),10)])
canvas = FigureCanvas(f)
response = HttpResponse(content_type='image/png')
canvas.print_png(response)
return response
def plotPSD(lcInt, shortExp,**kwargs):
'''
plot power spectral density of lc
return frequencies and powers from periodogram
'''
freq = 1.0/shortExp
f, p = periodogram(lcInt,fs = 1./shortExp)
plt.plot(f,p/np.max(p),**kwargs)
plt.xlabel(r"Frequency (Hz)",fontsize=14)
plt.xscale('log')
plt.ylabel(r"Normalized PSD",fontsize=14)
plt.yscale('log')
plt.title(r"Lightcurve Power Spectrum",fontsize=14)
plt.show()
return f,p
def plot_noise(self, apsize):
logging.info('Plotting noise budget')
p = self.p.aperture[apsize]
plt.scatter(p.medmag, p.rms)
plt.plot(p.mags_model, p.sigmastar_model, label='Star noise')
plt.plot(p.mags_model, p.sigmasky_model, label='Sky noise')
plt.plot(p.mags_model, p.sigmaread_model, label='Readout noise')
plt.plot(p.mags_model, p.sigscint_model, label='Scint. noise')
plt.yscale('log')
plt.plot(p.mags_model, p.sigtot_model, label='Tot. noise')
plt.legend(loc=4)
plt.savefig(os.path.join(self.plot_out, 'noise_budget.pdf'))
plt.show()
def show_reverseattn(trange_gaincal, trange_other=None, first_attn_base=5, second_attn_base=3, corrected_attns=False):
# This routine returns the same things as get_reverseattn, but it also graphs the three different "tsys"
# variables. It interactively asks the user which antenna, which polarization, and which frequency they
# wish to view. Once it creates a graph, it will ask the user whether they wish to see more graphs or not.
# The first parameter it takes should be a GAINCALTEST trange, which may be located with find_gaincal,
# The second parameter it takes should be a FEATTNTEST trange, which may be located with find_gaincal as
# well. It may or may not take a third parameter. Any form of file that was recorded by the system from
# from the antennas may be inserted here, whether a flare or just quiet sun data.
#
# PLEASE NOTE: ANY trange with data may be used as "trange_gaincal", use a trange from a GAINCALTEST and the other
# file as "trange_other" if you want the noise to be calibrated from the GAINCALTEST file, which will most likely
# be more recent than the FEATTNTEST file it would otherwise take the noise from.
tsys_attn_noise_corrected, tsys_noise_corrected, tsys = get_reverseattn(trange_gaincal, trange_other, first_attn_base, second_attn_base, corrected_attns)
plotting_ = True
while plotting_ == True:
print ' '
antenna = input('Which antenna would you like to see? You can choose from 1 to 8 : ')
print ' '
polarization = input('Which polarization would you like to see? Enter 0 for x, and 1 for y: ')
print ' '
channel = input('Which channel would you like to see? You can choose from channel 0 to ' +str(tsys_attn_noise_corrected.shape[2]) + ' : ')
antenna = int(antenna)-1
polarization = int(polarization)
channel = int(channel)
plt.figure()
tsys_ = plt.plot(tsys[antenna, polarization, channel, :], label='tsys')
tsys_noise_corrected_ = plt.plot(tsys_noise_corrected[antenna, polarization, channel, :], label='noise corrected tsys')
tsys_attn_noise_corrected_ = plt.plot(tsys_attn_noise_corrected[antenna, polarization, channel, :], label='noise and attn corrected tsys')
plt.yscale('log')
fontP = FontProperties()
fontP.set_size('small')
plt.legend(prop = fontP)
plt.show()
print ' '
yesorno = input('Would you like to see another plot? Please answer 0 for yes or 1 for no. ')
if yesorno == 0:
pass
else:
print ' '
print 'See you next time!'
plotting_ = False
return tsys_attn_noise_corrected, tsys_noise_corrected, tsys
def compute_prior_factor(size=64, gamma=0.5, alpha=1, do_plot=False):
file_name = os.path.join(data_dir, "CelebA_%s_prior_prob_smoothed.npy" % size)
prior_prob_smoothed = np.load(file_name)
u = np.ones_like(prior_prob_smoothed)
u = u / np.sum(1.0 * u)
prior_factor = (1 - gamma) * prior_prob_smoothed + gamma * u
prior_factor = np.power(prior_factor, -alpha)
# renormalize
prior_factor = prior_factor / (np.sum(prior_factor * prior_prob_smoothed))
file_name = os.path.join(data_dir, "CelebA_%s_prior_factor.npy" % size)
np.save(file_name, prior_factor)
if do_plot:
plt.plot(prior_factor)
plt.yscale("log")
plt.show()
def bargraph_from_dict(h_dict, group_labels, log=False, colors=None, save=None,
title=None, xlabel=None, ylabel=None, legend=False, loc='best'):
bars = len(h_dict) + 2 # Add one for a space between groups
groups = len(group_labels)
print h_dict
for v in h_dict.values():
if len(v) != groups:
raise ValueError("h_dict must contain sequences with the same number as labels")
plt.cla()
plts = []
if log:
plt.yscale("log")
if colors == "greyscale":
colors = [str(i*(.75/len(h_dict))) for i in xrange(len(h_dict))]
elif colors is None or colors == "spectrum":
colors = "rygbcmk"
keys = sorted(h_dict.keys())
for n, k in enumerate(keys):
plts.append(plt.bar(range(n+1, bars*groups+1, bars), h_dict[k],
color=colors[n % len(colors)]))
plt.xticks([1 + i*bars + bars/2 for i in xrange(groups)], group_labels)
if log:
ymin, ymax = plt.ylim()
ymin = min(reduce(lambda acc, x: acc+x, h_dict.values(), []))/2.0
plt.ylim((ymin, ymax))
if legend:
plt.legend( [p[0] for p in plts], [k.replace('_', '\_') for k in keys], loc=loc)
if title:
plt.title(title)
if ylabel:
plt.ylabel(ylabel)
if xlabel:
plt.xlabel(xlabel)
if save is not None:
plt.savefig(save)
def analyze_neuron(cell, mapping, Mea, signal_range):
start_t, stop_t = signal_range
start_t_ixd = np.argmin(np.abs(cell.tvec - start_t))
stop_t_ixd = np.argmin(np.abs(cell.tvec - stop_t))
t_array = cell.tvec[start_t_ixd:stop_t_ixd]
imem = cell.imem[:, start_t_ixd:stop_t_ixd]
amps_at_comps = find_amplitude_at_comp(imem, debug=False)
comp_dist_from_soma = find_comps_dist_from_soma(cell)
#pl.figure()
#pl.plot(comp_dist_from_soma, amps_at_comps, 'o')
#pl.show()
for elec in xrange(Mea.n_elecs):
#if not elec == 110:
# continue
print elec
comp_dist_from_elec = find_comp_dist_from_elec(elec, cell, Mea)
comp_impact_on_elec = mapping[elec,:]*amps_at_comps
if 0:
print np.argmax(comp_dist_from_elec)
print np.argmin(comp_dist_from_elec)
pl.figure()
pl.axis('equal')
pl.plot(cell.zmid/1000, cell.ymid/1000, 'o')
pl.plot(Mea.elec_z, Mea.elec_y, 'o')
pl.plot(cell.zmid[381]/1000, cell.ymid[381]/1000, 'D')
pl.plot(cell.zmid[791]/1000, cell.ymid[791]/1000, 'D')
pl.plot(Mea.elec_z[elec], Mea.elec_y[elec], 'x')
pl.show()
pl.plot(comp_dist_from_elec, comp_impact_on_elec, 'o')
pl.yscale('log')
pl.xscale('log')
pl.title('Electrode at distal tuft dendrite, when spike originates in tuft,\n'\
+'and does not propagate to soma.')
pl.xlabel('Compartment distance from electrode [mm]')
pl.ylabel('Compartment impact on electrode [uV]')
pl.show()
def plot_results(list_log, to_plot="losses"):
list_color = [u'#E24A33',
u'#348ABD',
u'#FBC15E',
u'#777777',
u'#988ED5',
u'#8EBA42',
u'#FFB5B8']
plt.figure()
for idx, log in enumerate(list_log):
with open(log, "r") as f:
d = json.load(f)
experiment_name = d["experiment_name"]
color = list_color[idx]
plt.plot(d["train_%s" % to_plot],
color=color,
linewidth=3,
label="Train %s" % experiment_name)
plt.plot(d["val_%s" % to_plot],
color=color,
linestyle="--",
linewidth=3,)
plt.ylabel(to_plot, fontsize=20)
if to_plot == "losses":
plt.yscale("log")
if to_plot == "accs":
plt.ylim([0, 1.1])
plt.xlabel("Number of epochs", fontsize=20)
plt.title("%s experiment" % dataset, fontsize=22)
plt.legend(loc="best")
plt.tight_layout()
plt.savefig("./figures/%s_results_%s.png" % (dataset, to_plot))
plt.show()
def plotter(infiles, numcases, labels, linestyles, colors, savefile, ylog=False):
"""Assuming we have output from CORBITS examples,
creates plots of Transit Probability vs Mean Mutual Inclination.
"""
plt.figure()
plt.xlabel('Mean Mutual Inclination [Degrees]', fontsize=16)
plt.ylabel('Transit Probability', fontsize=16)
if numcases == 1:
data = np.loadtxt(infiles[0])
mmi = data[:,0]
tp = data[:,1]
plt.plot(mmi, tp, linestyle=linestyles[0], color=colors[0], linewidth=4)
else:
for i in range(numcases):
data = np.loadtxt(infiles[i])
mmi = data[:,0]
tp = data[:,1]
plt.plot(mmi, tp, linestyle=linestyles[i], color=colors[i], linewidth=4, label=labels[i])
if numcases != 1:
plt.legend(loc='best')
if ylog == True:
plt.yscale('log')
plt.savefig(savefile, format='pdf')
return
def plot_tec(start=None, finish=None, ax=None, x_axis='EPHEMERIS_TIME',
fmt='k.', bad_fmt='r.', mew=0., labels=True):
"""Quickie that plots TEC in TECU"""
if ax is None:
ax = plt.gca()
else:
plt.sca(ax)
if x_axis == 'EPHEMERIS_TIME':
if start is None:
start, finish = plt.xlim()
else:
plt.xlim(start, finish)
t = read_tec(start, finish)
good = t['FLAG'] == True
plt.plot(t[x_axis][good], t['TEC'][good] / TECU_electrons_per_m2, fmt, mew=mew)
plt.plot(t[x_axis][~good], t['TEC'][~good] / TECU_electrons_per_m2, bad_fmt, mew=mew)
plt.xlabel(x_axis)
plt.ylabel('TEC / TECU')
plt.yscale('log')
args = parser.parse_args()
# NOTE: For debugging purposes only
# rates = DefaultRates
# fracs = Fracs()
# times = TimeRange(0, 100)
# param = Params(AGES[COUNTRY], POPDATA[make_key(COUNTRY, REGION)], times, rates, fracs)
# model = trace_ages(solve_ode(param, init_pop(param.ages, param.size, 1)))
res = fit_population(args.key)
model = trace_ages(
solve_ode(
res['params'],
init_pop(res['params'].ages, res['params'].size,
res['initialCases'])))
tp = res['params'].time - JAN1_2020
plt.figure()
plt.plot(tp, res['data'][Sub.T], 'o')
plt.plot(tp, model[:, Sub.T])
plt.plot(tp, res['data'][Sub.D], 'o')
plt.plot(tp, model[:, Sub.D])
plt.plot(tp, model[:, Sub.I])
plt.plot(tp, model[:, Sub.R])
plt.yscale('log')
def drawSubplots():
linearreg_true, linearreg_pred = read_csv("LinearRegression_results.csv")
rfr_true, rfr_pred = read_csv("RandomForestRegressor_results.csv")
arima_true, arima_pred = read_csv("ARIMA_results.csv")
rnn_true, rnn_pred = read_csv("RNN_results.csv")
size = 10
fig = plt.figure()
fig = plt.figure(dpi=100, figsize=(20, 7))
plt.subplot(221)
days = range(len(linearreg_true))
plt.plot(days, linearreg_true, color='r', label='truth sales')
plt.plot(days, linearreg_pred, color='b', label='pred sales')
plt.yscale('log')
plt.xlabel("days")
plt.ylabel("sales")
plt.legend(loc='upper left', frameon=False)
plt.title('Linear Regression', fontsize=size)
plt.subplot(222)
days = range(len(rfr_true))
plt.plot(days, rfr_true, color='r')
plt.plot(days, rfr_pred, color='b')
plt.yscale('log')
plt.xlabel("days")
plt.ylabel("sales")
# plt.legend(loc='upper left', frameon=False)
plt.title('Random Forest Regressor', fontsize=size)
plt.subplot(223)
days = range(len(arima_true))
plt.plot(days, arima_true, color='r')
plt.plot(days, arima_pred, color='b')
plt.yscale('log')
plt.xlabel("days")
plt.ylabel("sales")
# plt.legend(loc='upper left', frameon=False)
plt.title('ARIMA', fontsize=size)
plt.subplot(224)
days = range(len(rnn_true))
plt.plot(days, rnn_true, color='r')
plt.plot(days, rnn_pred, color='b')
plt.yscale('log')
plt.xlabel("days")
plt.ylabel("sales")
# plt.legend(loc='upper left', frameon=False)
plt.title('LSTM', fontsize=size)
plt.savefig("store 285 subplots.png",
format='png',
bbox_inches='tight',
transparent=False)
plt.show()
suffix = ['_CH_rate', '_CC_rate' ]
s =2
for k in range(0, len(comp)):
plt.figure(figsize=(8,4))
for i in range(0, len(names)):
rate = np.abs(cf.get_rate(results[names[i]][comp[k]],
results[names[i]]['time'][2] - results[names[i]]['time'][1],
step = s ))
#print(len(rate))
plt.plot(results[names[i]]['time'][::s],
rate,
ls=linetype[i], label = label[i])
plt.title(titles[k])
plt.xlabel('Time (s)')
plt.ylabel('Rate (mol/s)')
plt.yscale("log")
plt.legend()
plt.savefig(fpath + fname + suffix[k])
plt.show()
#plt.savefig(fpath + fname + '_CH_rate')
#%% Degree of carbonation
mm_CH = 74
mm_CC = 100
mass0 = results[names[0]]['portlandite'][0]*mm_CH + results[names[0]]['calcite'][0]*mm_CC
mass_f = results[names[0]]['portlandite'][-1]*mm_CH + results[names[0]]['calcite'][-1]*mm_CC
plt.figure(figsize=(8,4))
for i in range(0, len(names)):
d = ((np.array(results[names[i]]['portlandite'])*mm_CH + \
print rslt
# Asymmetry parameter for the same case
rslt = spam.asy_calc(Dp=200., wl=550., m=np.complex(1.54, 0.02))
print rslt
# Aplitude scattering matrix components S1 and S2 for the same case
rslt = spam.S12_calc(Dp=200., wl=550., m=np.complex(1.54, 0.02))
print rslt
# Normalized scattering phase function for the same case at 0 degrees
rslt = spam.spf(0., Dp=200., wl=550., m=np.complex(1.54, 0.02))
print rslt
# Same, but for a list of angles that can be plotted
angle_array = np.array(range(181))
rslt = spam.spf(angle_array, Dp=200., wl=550., m=np.complex(1.54, 0.02))
pl.figure()
pl.yscale('log')
pl.plot(angle_array, rslt)
# Calculate Legendre coefficients for the same case
# set nc to integer to return that number of coeffs, or None for automatic
rslt = spam.legendre_coeffs(Dp=200.,
wl=550.,
m=np.complex(1.54, 0.02),
nc=None)
print rslt
def test_compare_with_reference_phenomp(self):
import phenom
import numpy as np
m1 = 16.
m2 = 12.
chi1x = 0.6
chi1y = 0.
chi1z = 0.
chi2x = 0.
chi2y = 0.
chi2z = 0.
f_min = 30.
fRef = 30.
delta_f = 1 / 8.
inclination = np.pi / 8.
input_params = {}
input_params.update({'m1': m1})
input_params.update({'m2': m2})
input_params.update({'chi1x': chi1x})
input_params.update({'chi1y': chi1y})
input_params.update({'chi1z': chi1z})
input_params.update({'chi2x': chi2x})
input_params.update({'chi2y': chi2y})
input_params.update({'chi2z': chi2z})
input_params.update({'f_min': f_min})
input_params.update({'fRef': fRef})
input_params.update({'inclination': inclination})
input_params.update({'delta_f': delta_f})
phenp_new = phenom.PhenomP(**input_params)
f_new = phenp_new.flist_Hz
a_new = np.absolute(phenp_new.hp)
p_new = np.unwrap(np.angle(phenp_new.hp))
f, a, p = np.loadtxt('./phenp_ref.dat').T
try:
self.assertTrue(len(f) == len(f_new))
except:
raise ValueError(
'new frequency series does not have the same length as the reference frequency series'
)
tol = 6
try:
np.testing.assert_almost_equal(a, a_new, tol)
except:
import matplotlib
import matplotlib.pylab as plt
plt.figure()
plt.plot(np.abs(a - a_new))
plt.yscale('log')
plt.savefig('./comp_amp.png')
raise ValueError(
'new amplitude does not equal the reference amplitude')
try:
np.testing.assert_almost_equal(p, p_new, tol)
except:
import matplotlib
import matplotlib.pylab as plt
plt.figure()
plt.plot(np.abs(p - p_new))
plt.yscale('log')
plt.savefig('./comp_phase.png')
raise ValueError('new phase does not equal the reference phase')
def plot(self):
if not hasattr(self, 't'): self._construct_calibration_array()
if not hasattr(self, 'cal'): self.calibrate()
# plt.close('all')
plt.figure()
plt.plot(self.x, self.ly, 'k.')
plt.xlabel('Ionogram Signal')
plt.ylabel(r'$log_10 n_e / cm^{-3}$')
plt.hlines(np.log10(150.), plt.xlim()[0], plt.xlim()[1], color='green',
linestyles='dashed')
for i in range(self.cal.shape[1]):
c = self.cal[:,i]
print(c)
plt.plot((c[0], c[0]), (c[1]-c[2], c[1]+c[2]), 'r-')
plt.plot(c[0], c[1], 'r.')
p = np.polyfit(self.x, self.ly, 10)
x = np.arange(plt.xlim()[0], plt.xlim()[1], 0.01)
plt.plot(x, np.poly1d(p)(x), 'b-')
plt.figure()
dists = np.empty_like(self.t)
sigmas = np.empty_like(self.t)
for i in range(self.t.shape[0]):
# if i % 10 != 0: continue
val = 10.**np.interp(self.x[i], self.cal[0], self.cal[1])
err = 10.**np.interp(self.x[i], self.cal[0], self.cal[2])
plt.plot(self.y[i],val,'k.')
plt.plot((self.y[i],self.y[i]), (val-err, val+err), 'k-')
dists[i] = self.y[i] - val
sigmas[i] = np.abs(np.log10(dists[i])/np.log10(err))
dists[i] /= self.y[i]
x = np.array((10., 1E4))
plt.plot(x,x, 'r-')
plt.plot(x, x*2., 'r-')
plt.plot(x, x*.5, 'r-')
plt.yscale('log')
plt.xscale('log')
plt.figure()
plt.hist(np.abs(dists), bins=20, range=(0., 4.))
# plt.figure()
# plt.hist(sigmas, bins=20)
dists = np.abs(dists)
# some statistics:
s = 100. / float(self.t.shape[0])
print()
print('%f%% with relative error < 0.1' % (np.sum(dists < 0.1) * s))
print('%f%% with relative error < 0.5' % (np.sum(dists < 0.5) * s))
print('%f%% with relative error < 1.0' % (np.sum(dists < 1.0) * s))
print('%f%% with relative error < 2.0' % (np.sum(dists < 2.0) * s))
print()
print('%f%% with sigma < 1.0' % (np.sum(sigmas < 1.0) * s))
print('%f%% with sigma < 2.0' % (np.sum(sigmas < 2.0) * s))
print('%f%% with sigma < 4.0' % (np.sum(sigmas < 4.0) * s))
plt.show()
def createPlot():
# collect the data
avBitsExp = []
z5PerBitsExp = []
z95PerBitsExp = []
avBitsIndices = []
z5PerBitsIndices = []
z95PerBitsIndices = []
n = [
10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 200, 300, 400, 500, 600, 700,
800, 900, 1000, 2000, 3000, 4000, 5000
]
for exp in expTypeNames:
avBitsExp.append(float(d[exp + 'medianBitsExp'][0]))
z5PerBitsExp.append(float(d[exp + '5PerBitsExp'][0]))
z95PerBitsExp.append(float(d[exp + '95PerBitsExp'][0]))
# avBitsTruncated.append(float(d[exp+'avBitsTruncated'][0]))
avBitsIndices.append(float(d[exp + 'medianBitsIndices'][0]))
z5PerBitsIndices.append(float(d[exp + '5PerBitsIndices'][0]))
z95PerBitsIndices.append(float(d[exp + '95PerBitsIndices'][0]))
# converting to np arrays
n = np.array(n)
avBitsExp = np.array(avBitsExp)
avBitsIndices = np.array(avBitsIndices)
avBitsExp[avBitsExp == -1.0] = np.nan
avBitsIndices[avBitsIndices == -1.0] = np.nan
# datapoints that are not involved in the curve calc (NCD - Non Curve Data)
nNCD = np.concatenate((n[:9], n[19:]))
avBitsExpNCD = np.concatenate((avBitsExp[:9], avBitsExp[19:]))
avBitsIndicesNCD = np.concatenate((avBitsIndices[:9], avBitsIndices[19:]))
# datapoints involved in creating the curves
nCurveData = n[9:19]
avBitsExpCurveData = avBitsExp[9:19]
z5PerBitsExpCurveData = z5PerBitsExp[9:19]
z95PerBitsExpCurveData = z95PerBitsExp[9:19]
avBitsIndicesCurveData = avBitsIndices[9:19]
z5PerBitsIndicesCurveData = z5PerBitsIndices[9:19]
z95PerBitsIndicesCurveData = z95PerBitsIndices[9:19]
# curve function
newx = np.logspace(0, 5, 250)
def func(x, a, b, c):
return a + b * x + c * x * x
poptExpAv, _ = curve_fit(func, np.log(nCurveData),
np.log(avBitsExpCurveData))
poptExp5, _ = curve_fit(func, np.log(nCurveData),
np.log(z5PerBitsExpCurveData))
poptExp95, _ = curve_fit(func, np.log(nCurveData),
np.log(z95PerBitsExpCurveData))
poptIndAv, _ = curve_fit(func, np.log(nCurveData),
np.log(avBitsIndicesCurveData))
poptInd5, _ = curve_fit(func, np.log(nCurveData),
np.log(z5PerBitsIndicesCurveData))
poptInd95, _ = curve_fit(func, np.log(nCurveData),
np.log(z95PerBitsIndicesCurveData))
print(poptExpAv)
print(poptIndAv)
# plotting
plt.figure(facecolor='w', edgecolor='k', figsize=(7, 5))
plt.xlabel("$n$")
plt.ylabel("bits required")
expColor = 'orangered'
plt.plot(nCurveData,
avBitsExpCurveData,
'o',
color=expColor,
label="Exponential weight approach")
plt.plot(nNCD, avBitsExpNCD, 'o', color=expColor, fillstyle='none')
plt.plot(newx,
np.exp(
func(np.log(newx), poptExpAv[0], poptExpAv[1], poptExpAv[2])),
'-',
color=expColor)
plt.fill_between(
newx,
np.exp(func(np.log(newx), poptExp5[0], poptExp5[1], poptExp5[2])),
np.exp(func(np.log(newx), poptExp95[0], poptExp95[1], poptExp95[2])),
color=expColor,
alpha=.5)
indColor = 'seagreen'
plt.plot(nCurveData,
avBitsIndicesCurveData,
'o',
color=indColor,
label="Vector-based weight approach")
plt.plot(nNCD, avBitsIndicesNCD, 'o', color=indColor, fillstyle='none')
plt.plot(newx,
np.exp(
func(np.log(newx), poptIndAv[0], poptIndAv[1], poptIndAv[2])),
'-',
color=indColor)
plt.fill_between(
newx,
np.exp(func(np.log(newx), poptInd5[0], poptInd5[1], poptInd5[2])),
np.exp(func(np.log(newx), poptInd95[0], poptInd95[1], poptInd95[2])),
color=indColor,
alpha=.5)
# horizontal lines
gb = 8589934592
mb100 = 838860800
mb10 = 83886080
mb1 = 8388608
ax = plt.subplot()
ax.axhline(y=mb1,
xmin=0.0,
xmax=1.0,
color='gray',
linestyle='-',
linewidth=.5)
ax.axhline(y=mb10,
xmin=0.0,
xmax=1.0,
color='gray',
linestyle='-',
linewidth=.5)
ax.axhline(y=mb100,
xmin=0.0,
xmax=1.0,
color='gray',
linestyle='-',
linewidth=.5)
ax.axhline(y=gb,
xmin=0.0,
xmax=1.0,
color='gray',
linestyle='-',
linewidth=.5)
ax.annotate('1MB',
xy=(1.5, mb1 + 1000000),
xytext=(1.5, mb1 + 1000000),
color='gray')
ax.annotate('10MB',
xy=(1.5, mb10 + 10000000),
xytext=(1.5, mb10 + 10000000),
color='gray')
ax.annotate('100MB',
xy=(1.5, mb100 + 100000000),
xytext=(1.5, mb100 + 100000000),
color='gray')
ax.annotate('1GB',
xy=(1.5, gb + 1000000000),
xytext=(1.5, gb + 1000000000),
color='gray')
# general plot info
plt.legend(loc='lower right')
ax = plt.subplot()
# ax.spines["right"].set_visible(False)
# ax.spines["top"].set_visible(False)
ax.set_xlim(1, 100000)
plt.xscale('log')
plt.yscale('log')
# plt.grid()
ax.xaxis.grid(True)
# plt.show()
plt.tight_layout()
plt.savefig("./stats/motivation/plot_space.pdf")
def plot_fitted_xprofiles(self):
# fitted model
for solution_idx, solution_val in enumerate(
self.pickle_file['solutions']):
fig = plt.figure(figsize=(7, 7 / phi))
ax = fig.add_subplot(111)
N = len(self.pickle_file['params']['atm_active_gases'] +
self.pickle_file['params']['atm_inactive_gases'])
cols_mol = {}
for mol_idx, mol_val in enumerate(
self.pickle_file['params']['atm_active_gases']):
cols_mol[mol_val] = self.cmap(float(mol_idx) / N)
prof = solution_val['active_mixratio_profile'][mol_idx]
plt.plot(prof[:, 1],
prof[:, 0] / 1e5,
color=cols_mol[mol_val],
label=mol_val)
#plt.fill_betweenx(prof[:,0]/1e5, prof[:,1]-prof[:,2], prof[:,1]+prof[:,2], color=self.cmap(float(mol_idx)/N), alpha=0.5)
for mol_idx, mol_val in enumerate(
self.pickle_file['params']['atm_inactive_gases']):
cols_mol[mol_val] = self.cmap(
float(mol_idx + len(self.pickle_file['params']
['atm_inactive_gases'])) / N)
prof = solution_val['inactive_mixratio_profile'][mol_idx]
plt.plot(prof[:, 1],
prof[:, 0] / 1e5,
color=cols_mol[mol_val],
label=mol_val)
#plt.fill_betweenx(prof[:,0]/1e5, prof[:,1]-prof[:,2], prof[:,1]+prof[:,2], color=self.cmap(float(mol_idx)/N), alpha=0.5)
# add input spectrum param if available
if 'SPECTRUM_db' in self.pickle_file:
for mol_idx, mol_val in enumerate(self.pickle_file['SPECTRUM_db']
['params']['atm_active_gases']):
prof = self.pickle_file['SPECTRUM_db']['data'][
'active_mixratio_profile'][mol_idx]
if mol_val in cols_mol:
plt.plot(prof[:, 1],
prof[:, 0] / 1e5,
color=cols_mol[mol_val],
ls='dashed')
else:
plt.plot(prof[:, 1],
prof[:, 0] / 1e5,
color=self.cmap(
float(
len(self.pickle_file['params']
['atm_active_gases']) +
len(self.pickle_file['params']
['atm_inactive_gases']) + mol_idx) /
N),
label=mol_val,
ls='dashed')
for mol_idx, mol_val in enumerate(
self.pickle_file['SPECTRUM_db']['params']
['atm_inactive_gases']):
prof = self.pickle_file['SPECTRUM_db']['data'][
'inactive_mixratio_profile'][mol_idx]
if mol_val in cols_mol:
plt.plot(prof[:, 1],
prof[:, 0] / 1e5,
color=cols_mol[mol_val],
ls='dashed')
else:
plt.plot(prof[:, 1],
prof[:, 0] / 1e5,
color=self.cmap(
float(
len(self.pickle_file['params']
['atm_active_gases']) +
len(self.pickle_file['params']
['atm_inactive_gases']) + mol_idx) /
N),
label=mol_val,
ls='dashed')
plt.gca().invert_yaxis()
plt.yscale('log')
plt.xscale('log')
plt.xlim(1e-12, 3)
plt.xlabel('Mixing ratio')
plt.ylabel('Pressure (bar)')
plt.tight_layout()
box = ax.get_position()
ax.set_position([box.x0, box.y0, box.width * 0.8, box.height])
ax.legend(loc='center left',
bbox_to_anchor=(1, 0.5),
ncol=1,
prop={'size': 11},
frameon=False)
if self.title:
plt.title(self.title, fontsize=14)
plt.savefig(
os.path.join(
self.out_folder,
'%s%s_mixratio.pdf' % (self.prefix, self.retrieval_type)))
def plot_variance_reduction_cir_process(savefig=False, plot_from_json=True):
poly_orders = {'linear': 1, 'cubic': 3}
poly_markers = (i for i in ['s', 'd'])
markers = {
**{
'exact': 'o',
'euler_maruyama': 'v'
},
**{k: next(poly_markers)
for k in poly_orders}
}
if plot_from_json:
with open('variance_reduction_cir_process.json', "r") as input_file:
results = json.load(input_file)
results = {
k: {float(x): y
for x, y in v.items()}
for k, v in results.items()
}
else:
deltas = [0.5**i for i in range(8)]
params = {'kappa': 0.5, 'theta': 1.0, 'sigma': 1.0}
nu = 4.0 * params['kappa'] * params['theta'] / (params['sigma']**2)
approximations = [
construct_inverse_non_central_chi_squared_interpolated_polynomial_approximation(
dof=nu, polynomial_order=poly_order) for poly_order in [1, 3]
]
results = {
k: {}
for k in ['exact', 'euler_maruyama'] + list(poly_orders.keys())
}
time_per_level = 5.0
paths_min = 64
for dt in progressbar(deltas):
_, elapsed_time_per_path = time_function(
produce_cox_ingersoll_ross_paths)(dt, approximations, **params)
paths_required = int(time_per_level / elapsed_time_per_path)
if paths_required < paths_min:
print("More time required for dt={}".format(dt))
break
exacts, euler_maruyamas, linears, cubics = [
[None for i in range(paths_required)] for j in range(4)
]
for path in range(paths_required):
x_euler_maruyama, x_exact, x_linear, x_cubic = produce_cox_ingersoll_ross_paths(
dt, approximations, **params)
exacts[path] = x_exact
euler_maruyamas[path] = x_exact - x_euler_maruyama
linears[path] = x_exact - x_linear
cubics[path] = x_exact - x_cubic
exacts, euler_maruyamas, linears, cubics = [[
j**2 for j in i
] for i in [exacts, euler_maruyamas, linears, cubics]]
for name, values in {
'exact': exacts,
'euler_maruyama': euler_maruyamas,
'linear': linears,
'cubic': cubics
}.items():
mean = np.mean(values)
std = np.std(values) / (len(values)**0.5)
results[name][dt] = [mean, std]
plt.clf()
for name in results:
x, y = zip(*results[name].items())
y, y_std = list(zip(*y))
y_error = 1 * np.array(y_std)
plt.errorbar(x, y, y_error, None, 'k{}:'.format(markers[name]))
plt.xscale('log', basex=2)
plt.yscale('log', basey=2)
plt.xticks(x)
plt.ylim(2**-25, 2**2)
plt.yticks([2**-i for i in range(0, 30, 5)])
plt.xlabel(r'Time increment $\delta$')
plt.ylabel('Variance')
if savefig:
plt.savefig('variance_reduction_cir_process.pdf',
format='pdf',
bbox_inches='tight',
transparent=True)
if not plot_from_json:
with open('variance_reduction_cir_process.json',
"w") as output_file:
output_file.write(json.dumps(results, indent=4))
def compute_prior_prob_v1(image_files, is_resize, width, height, do_plot,
pts_in_hull_path, prior_prob_path, ab_hist_path):
"""
Compute color prior probabilities for pts in hull
Reference: https://github.com/foamliu/Colorful-Image-Colorization/blob/master/class_rebal.py
Usage:
df_data = pd.read_hdf(os.path.join(dataset_dir, "DIV2K", "div2k.hdf5"), "data")
list_types = ["'train'"]
df_select_data = df_data.query("type in [" + ",".join(list_types) + "]")
image_dir = os.path.join(dataset_dir, "DIV2K").replace("\\", "/")
image_files = image_dir + "/" + df_select_data["path"].values
image_files[0:3], len(image_files)
info = dict(
image_files = image_files,
pts_in_hull_path = os.path.join(module_dir, "data", "pts_in_hull.npy"),
prior_prob_path = os.path.join(module_dir, "data", "prior_prob_train_div2k.npy"),
ab_hist_path = os.path.join(data_dir, "preprocessing", "DIV2K", "ab_hist_train_div2k.npy"),
is_resize = False,
width = 112,
height = 112,
do_plot = True
)
locals().update(**info)
prior_prob = compute_prior_prob(**info)
"""
# Load ab image
ab_hist = np.zeros((256, 256), dtype=np.uint64)
for image_path in tqdm.tqdm(image_files):
result = read_image(image_path,
is_resize=is_resize,
width=width,
height=height)
I_ab = result["res_img_Lab"][:, :,
1:] if is_resize == True else result[
"org_img_Lab"][:, :, 1:]
I_ab = I_ab.reshape(-1, 2).astype(np.uint)
(ab_vals, ab_cnts) = np.unique(I_ab, return_counts=True, axis=0)
ab_hist[ab_vals[:, 0], ab_vals[:, 1]] += ab_cnts.astype(np.uint64)
# for
# Load the gamut points location
q_ab = np.load(pts_in_hull_path)
if do_plot:
plt.figure(figsize=(8, 8))
gs = gridspec.GridSpec(1, 1)
ax = plt.subplot(gs[0])
for i in range(q_ab.shape[0]):
ax.scatter(q_ab[:, 0], q_ab[:, 1])
ax.annotate(str(i), (q_ab[i, 0], q_ab[i, 1]), fontsize=6)
ax.set_xlim([-110, 110])
ax.set_ylim([-110, 110])
# for
plt.title("Prior Distribution in ab space\n", fontsize=16)
plt.imshow(ab_hist.transpose(),
norm=LogNorm(),
cmap=plt.cm.jet,
extent=(-128, 127, -128, 127),
origin="uper")
plt.xlim([-120, 120])
plt.ylim([-120, 120])
plt.xticks(fontsize=12)
plt.yticks(fontsize=12)
plt.xlabel("b channel", fontsize=14)
plt.ylabel("a channel", fontsize=14)
plt.colorbar()
plt.show()
plt.clf()
plt.close()
# if
X_ab_ravel_h = np.vstack(np.nonzero(ab_hist)).T
X_ab_ravel_h = X_ab_ravel_h - 128
# Create nearest neighbord instance with index = q_ab
NN = 1
nearest = nn.NearestNeighbors(n_neighbors=NN,
algorithm='ball_tree').fit(q_ab)
# Find index of nearest neighbor for X_ab
dists, ind = nearest.kneighbors(X_ab_ravel_h)
# We now count the number of occurrences of each color
ind = np.ravel(ind)
counts = np.zeros(np.max(ind) + 1, np.uint64)
for idx, (a, b) in enumerate(X_ab_ravel_h):
counts[ind[idx]] = counts[ind[idx]] + ab_hist[(a + 128, b + 128)]
pass
# for
idxs = np.nonzero(counts)[0]
prior_prob = np.zeros((q_ab.shape[0]))
prior_prob[idxs] = counts[idxs]
# We turn this into a color probability
prior_prob = prior_prob / (1.0 * np.sum(prior_prob))
# Save
if prior_prob_path is not None:
save_dir = os.path.dirname(prior_prob_path)
if save_dir != "" and os.path.exists(save_dir) == False:
os.makedirs(save_dir)
np.save(prior_prob_path, prior_prob)
# if
# Save
if ab_hist_path is not None:
save_dir = os.path.dirname(ab_hist_path)
if save_dir != "" and os.path.exists(save_dir) == False:
os.makedirs(save_dir)
np.save(ab_hist_path, ab_hist)
# if
if do_plot:
plt.hist(prior_prob, bins=100)
plt.xlabel("Prior probability")
plt.ylabel("Frequency")
plt.yscale("log")
plt.show()
# if
return prior_prob, ab_hist
pass
def placementmass(massarr, z_ssb_arr, L_arr, beta_arr):
label1 = r"$\mathrm{h}^{-1} \mathrm{M}_{\odot}$"
if len(z_ssb_arr) > 1:
L = L_arr[0]
beta = beta_arr[0]
for z_ssb in z_ssb_arr:
d_vir = []
d_ta = []
d_c = []
rvirrta = []
avirata = []
variable = []
avir = []
acoll = []
delta_c = []
for M in massarr:
filename = "Numbers\Symmetron\Sym2\M%.15fz_ssb%.3fL%.3fbeta%.3f.txt" % (
np.log10(M), z_ssb, L, beta)
file = open(filename, "r")
i = 0
for j in file.readlines():
try:
value = float(j)
except ValueError:
break
if i == 0:
variable.append(value)
i += 1
elif i == 1:
d_vir.append(value)
i += 1
elif i == 2:
d_ta.append(value)
i += 1
elif i == 3:
d_c.append(value)
i += 1
elif i == 4:
rvirrta.append(value)
i += 1
elif i == 5:
avirata.append(value)
i += 1
elif i == 6:
avir.append(value)
i += 1
elif (i == 7):
acoll.append(value)
i += 1
elif i == 8:
delta_c.append(value)
i = 0
file.close()
seccondvar = r"$z_{\mathrm{ssb}} = %.1f$ " % z_ssb
mpl.figure(1)
mpl.plot(variable, d_vir, linewidth=0.75, label=seccondvar)
mpl.legend(fontsize=15)
mpl.xlabel(label1, fontsize=15)
mpl.ylabel(r"$\Delta_{vir}$", rotation=0, labelpad=20, fontsize=15)
mpl.xscale("log")
mpl.figure(2)
mpl.plot(variable, d_ta, linewidth=0.75, label=seccondvar)
mpl.legend(fontsize=15)
mpl.xlabel(label1, fontsize=15)
mpl.ylabel(r"$\Delta_{ta}$", fontsize=15, rotation=0, labelpad=20)
mpl.xscale("log")
mpl.figure(3)
mpl.plot(variable, rvirrta, linewidth=0.75, label=seccondvar)
mpl.legend()
mpl.xlabel(label1)
mpl.ylabel(r"$\frac{\tilde{R}_{vir}}{\tilde{R}_{ta}}$",
rotation=0,
labelpad=20)
mpl.xscale("log")
mpl.figure(4)
mpl.plot(variable, avirata, linewidth=0.75, label=seccondvar)
mpl.legend()
mpl.xlabel(label1)
mpl.ylabel(r"$\frac{a_{vir}}{a_{ta}}$", rotation=0, labelpad=20)
mpl.xscale("log")
mpl.figure(5)
mpl.plot(variable, avir, linewidth=0.75, label=seccondvar)
mpl.legend()
mpl.xlabel(label1)
mpl.ylabel(r"$a_{vir}$", rotation=0, labelpad=20)
mpl.xscale("log")
mpl.figure(6)
mpl.plot(variable, d_c, linewidth=0.75, label=seccondvar)
mpl.legend(fontsize=15)
mpl.xlabel(label1, fontsize=15)
mpl.ylabel(r"$\Delta_{c}$", rotation=0, labelpad=20, fontsize=15)
mpl.xscale("log")
mpl.figure(7)
mpl.plot(variable,
abs(np.array(acoll) - 1),
linewidth=0.75,
label=seccondvar)
mpl.legend()
mpl.xlabel(label1)
mpl.ylabel(r"$|a_c - 1|$", rotation=0, labelpad=20)
mpl.xscale("log")
mpl.figure(8)
mpl.plot(variable, delta_c, linewidth=0.75, label=seccondvar)
mpl.legend(fontsize=15)
mpl.xlabel(label1, fontsize=15)
mpl.ylabel(r"$\delta_i$", rotation=0, labelpad=20, fontsize=15)
mpl.xscale("log")
mpl.yscale("log")
mpl.figure(1)
mpl.axhline(y=295.618,
color="red",
linestyle="--",
linewidth=0.75,
label=r"$\Lambda$CDM")
mpl.legend()
mpl.title(r"$L = %.1f \mathrm{h}^{-1} \mathrm{Mpc}$, $\beta = %.1f$" %
(L, beta))
mpl.savefig("Figures\Symmetron\Zdvir.pdf", bbox_inches="tight")
mpl.clf()
mpl.figure(2)
mpl.axhline(y=7.085,
color="red",
linestyle="--",
linewidth=0.75,
label=r"$\Lambda$CDM")
mpl.legend()
mpl.title(r"$L = %.1f \mathrm{h}^{-1} \mathrm{Mpc}$, $\beta = %.1f$" %
(L, beta))
mpl.savefig("Figures\Symmetron\Zdta.pdf", bbox_inches="tight")
mpl.clf()
mpl.figure(3)
mpl.axhline(y=0.481722,
color="red",
linestyle="--",
linewidth=0.75,
label=r"$\Lambda$CDM")
mpl.legend()
mpl.title(r"$L = %.1f \mathrm{h}^{-1} \mathrm{Mpc}$, $\beta = %.1f$" %
(L, beta))
mpl.savefig("Figures\Symmetron\Zrvirrta.pdf", bbox_inches="tight")
mpl.clf()
mpl.figure(4)
mpl.axhline(y=1.66974,
color="red",
linestyle="--",
linewidth=0.75,
label=r"$\Lambda$CDM")
mpl.legend()
mpl.title(r"$L = %.1f \mathrm{h}^{-1} \mathrm{Mpc}$, $\beta = %.1f$" %
(L, beta))
mpl.savefig("Figures\Symmetron\Zavirata.pdf", bbox_inches="tight")
mpl.clf()
mpl.figure(5)
mpl.axhline(y=0.91889,
color="red",
linestyle="--",
linewidth=0.75,
label=r"$\Lambda$CDM")
mpl.legend()
mpl.title(r"$L = %.1f \mathrm{Mpc}$, $\beta = %.1f$" % (L, beta))
mpl.savefig("Figures\Symmetron\Zavir.pdf", bbox_inches="tight")
mpl.clf()
mpl.figure(6)
mpl.axhline(y=381.00185776,
color="red",
linestyle="--",
linewidth=0.75,
label=r"$\Lambda$CDM")
mpl.legend()
mpl.title(r"$L = %.1f \mathrm{h}^{-1}\mathrm{Mpc} $, $\beta = %.1f$" %
(L, beta))
mpl.savefig("Figures\Symmetron\Zdelta_c.pdf", bbox_inches="tight")
mpl.clf()
mpl.figure(7)
#mpl.axhline(y = 1, color = "red", linestyle = "--", linewidth = 0.75, label = r"$\Lambda$CDM")
mpl.legend()
mpl.title(r"$L = %.1f \mathrm{h}^{-1}\mathrm{Mpc} $, $\beta = %.1f$" %
(L, beta))
mpl.savefig("Figures\Symmetron\Za_c.pdf", bbox_inches="tight")
mpl.clf()
mpl.figure(8)
mpl.axhline(y=0.00059037,
color="red",
linestyle="--",
linewidth=0.75,
label=r"$\Lambda$CDM")
mpl.legend()
mpl.title(r"$L = %.1f \mathrm{h}^{-1}\mathrm{Mpc} $, $\beta = %.1f$" %
(L, beta))
mpl.savefig("Figures\Symmetron\Zdelta_i.pdf", bbox_inches="tight")
mpl.clf()
elif len(L_arr) > 1:
z_ssb = z_ssb_arr[0]
beta = beta_arr[0]
for L in L_arr:
d_vir = []
d_ta = []
d_c = []
rvirrta = []
avirata = []
variable = []
avir = []
acoll = []
delta_c = []
for M in massarr:
filename = "Numbers\Symmetron\Sym2\M%.15fz_ssb%.3fL%.3fbeta%.3f.txt" % (
np.log10(M), z_ssb, L, beta)
file = open(filename, "r")
i = 0
for j in file.readlines():
try:
value = float(j)
except ValueError:
break
if i == 0:
variable.append(value)
i += 1
elif i == 1:
d_vir.append(value)
i += 1
elif i == 2:
d_ta.append(value)
i += 1
elif i == 3:
d_c.append(value)
i += 1
elif i == 4:
rvirrta.append(value)
i += 1
elif i == 5:
avirata.append(value)
i += 1
elif i == 6:
avir.append(value)
i += 1
elif (i == 7):
acoll.append(value)
i += 1
elif i == 8:
delta_c.append(value)
i = 0
file.close()
seccondvar = r"$L = %.1f \mathrm{Mpc}\mathrm{h}^{-1}$ " % L
mpl.figure(1)
mpl.plot(variable, d_vir, linewidth=0.75, label=seccondvar)
mpl.legend(fontsize=15)
mpl.xlabel(label1, fontsize=15)
mpl.ylabel(r"$\Delta_{vir}$", rotation=0, labelpad=20, fontsize=15)
mpl.xscale("log")
mpl.figure(2)
mpl.plot(variable, d_ta, linewidth=0.75, label=seccondvar)
mpl.legend(fontsize=15)
mpl.xlabel(label1, fontsize=15)
mpl.ylabel(r"$\Delta_{ta}$", rotation=0, labelpad=20, fontsize=15)
mpl.xscale("log")
mpl.figure(3)
mpl.plot(variable, rvirrta, linewidth=0.75, label=seccondvar)
mpl.legend()
mpl.xlabel(label1)
mpl.ylabel(r"$\frac{\tilde{R}_{vir}}{\tilde{R}_{ta}}$",
rotation=0,
labelpad=20)
mpl.xscale("log")
mpl.figure(4)
mpl.plot(variable, avirata, linewidth=0.75, label=seccondvar)
mpl.legend()
mpl.xlabel(label1)
mpl.ylabel(r"$\frac{a_{vir}}{a_{ta}}$", rotation=0, labelpad=20)
mpl.xscale("log")
mpl.figure(5)
mpl.plot(variable, avir, linewidth=0.75, label=seccondvar)
mpl.legend()
mpl.xlabel(label1)
mpl.ylabel(r"$a_{vir}$", rotation=0, labelpad=20)
mpl.xscale("log")
mpl.figure(6)
mpl.plot(variable, d_c, linewidth=0.75, label=seccondvar)
mpl.legend(fontsize=15)
mpl.xlabel(label1, fontsize=15)
mpl.ylabel(r"$\Delta_{c}$", rotation=0, labelpad=20, fontsize=15)
mpl.xscale("log")
mpl.figure(7)
mpl.plot(variable,
abs(np.array(acoll) - 1),
linewidth=0.75,
label=seccondvar)
mpl.legend()
mpl.xlabel(label1)
mpl.ylabel(r"$|a_c - 1|$", rotation=0, labelpad=20)
mpl.xscale("log")
mpl.figure(8)
mpl.plot(variable, delta_c, linewidth=0.75, label=seccondvar)
mpl.legend()
mpl.xlabel(label1)
mpl.ylabel(r"$\delta_i$", rotation=0, labelpad=20)
mpl.xscale("log")
mpl.yscale("log")
mpl.figure(1)
mpl.axhline(y=295.6181,
color="red",
linestyle="--",
linewidth=0.75,
label=r"$\Lambda$CDM")
mpl.legend()
mpl.title(r"$z_{\mathrm{ssb}} = %.1f$, $\beta = %.1f$" % (z_ssb, beta))
mpl.savefig("Figures\Symmetron\Ldvir.pdf", bbox_inches="tight")
mpl.clf()
mpl.figure(2)
mpl.axhline(y=7.0985,
color="red",
linestyle="--",
linewidth=0.75,
label=r"$\Lambda$CDM")
mpl.legend(loc=2)
mpl.title(r"$z_{\mathrm{ssb}} = %.1f$, $\beta = %.1f $" %
(z_ssb, beta))
mpl.savefig("Figures\Symmetron\Ldta.pdf", bbox_inches="tight")
mpl.clf()
mpl.figure(3)
mpl.axhline(y=0.481722,
color="red",
linestyle="--",
linewidth=0.75,
label=r"$\Lambda$CDM")
mpl.legend()
mpl.savefig("Figures\Symmetron\Lrvirrta.pdf", bbox_inches="tight")
mpl.clf()
mpl.figure(4)
mpl.axhline(y=1.6697,
color="red",
linestyle="--",
linewidth=0.75,
label=r"$\Lambda$CDM")
mpl.legend()
mpl.title(r"$z_{\mathrm{ssb}} = %.1f$, $\beta = %.1f $" %
(z_ssb, beta))
mpl.savefig("Figures\Symmetron\Lavirata.pdf", bbox_inches="tight")
mpl.clf()
mpl.figure(5)
mpl.axhline(y=0.91889,
color="red",
linestyle="--",
linewidth=0.75,
label=r"$\Lambda$CDM")
mpl.legend()
mpl.title(r"$z_{\mathrm{ssb}} = %.1f$, $\beta = %.1f $" %
(z_ssb, beta))
mpl.savefig("Figures\Symmetron\Lavir.pdf", bbox_inches="tight")
mpl.clf()
mpl.figure(6)
mpl.axhline(y=381.00186,
color="red",
linestyle="--",
linewidth=0.75,
label=r"$\Lambda$CDM")
mpl.legend()
mpl.title(r"$z_{\mathrm{ssb}} = %.1f$, $\beta = %.1f$" % (z_ssb, beta))
mpl.savefig("Figures\Symmetron\Ldelta_c.pdf", bbox_inches="tight")
mpl.clf()
mpl.figure(7)
#mpl.axhline(y = 1, color = "red", linestyle = "--", linewidth = 0.75, label = r"$\Lambda$CDM")
mpl.legend()
mpl.title(r"$z_{\mathrm{ssb}} = %.1f$, $\beta = %.1f $" %
(z_ssb, beta))
mpl.savefig("Figures\Symmetron\La_c.pdf", bbox_inches="tight")
mpl.clf()
mpl.figure(8)
mpl.axhline(y=0.00059037,
color="red",
linestyle="--",
linewidth=0.75,
label=r"$\Lambda$CDM")
mpl.legend()
mpl.title(r"$z_{\mathrm{ssb}} = %.1f$, $\beta = %.1f $" %
(z_ssb, beta))
mpl.savefig("Figures\Symmetron\Ldelta_i.pdf", bbox_inches="tight")
mpl.clf()
elif len(beta_arr) > 1:
z_ssb = z_ssb_arr[0]
L = L_arr[0]
for beta in beta_arr:
d_vir = []
d_ta = []
d_c = []
rvirrta = []
avirata = []
variable = []
avir = []
acoll = []
delta_c = []
for M in massarr:
filename = "Numbers\Symmetron\Sym2\M%.15fz_ssb%.3fL%.3fbeta%.3f.txt" % (
np.log10(M), z_ssb, L, beta)
file = open(filename, "r")
i = 0
for j in file.readlines():
try:
value = float(j)
except ValueError:
break
if i == 0:
variable.append(value)
i += 1
elif i == 1:
d_vir.append(value)
i += 1
elif i == 2:
d_ta.append(value)
i += 1
elif i == 3:
d_c.append(value)
i += 1
elif i == 4:
rvirrta.append(value)
i += 1
elif i == 5:
avirata.append(value)
i += 1
elif i == 6:
avir.append(value)
i += 1
elif (i == 7):
acoll.append(value)
i += 1
elif i == 8:
delta_c.append(value)
i = 0
file.close()
seccondvar = r"$\beta= %.1f$ " % beta
mpl.figure(1)
mpl.plot(variable, d_vir, linewidth=0.75, label=seccondvar)
mpl.legend(fontsize=15)
mpl.xlabel(label1, fontsize=15)
mpl.ylabel(r"$\Delta_{vir}$", rotation=0, labelpad=20, fontsize=15)
mpl.xscale("log")
mpl.figure(2)
mpl.plot(variable, d_ta, linewidth=0.75, label=seccondvar)
mpl.legend(fontsize=15)
mpl.xlabel(label1, fontsize=15)
mpl.ylabel(r"$\Delta_{ta}$", rotation=0, labelpad=20, fontsize=15)
mpl.xscale("log")
mpl.figure(3)
mpl.plot(variable, rvirrta, linewidth=0.75, label=seccondvar)
mpl.legend()
mpl.xlabel(label1)
mpl.ylabel(r"$\frac{\tilde{R}_{vir}}{\tilde{R}_{ta}}$",
rotation=0,
labelpad=20)
mpl.xscale("log")
mpl.figure(4)
mpl.plot(variable, avirata, linewidth=0.75, label=seccondvar)
mpl.legend()
mpl.xlabel(label1)
mpl.ylabel(r"$\frac{a_{vir}}{a_{ta}}$", rotation=0, labelpad=20)
mpl.xscale("log")
mpl.figure(5)
mpl.plot(variable, avir, linewidth=0.75, label=seccondvar)
mpl.legend()
mpl.xlabel(label1)
mpl.ylabel(r"$a_{vir}$", rotation=0, labelpad=20)
mpl.xscale("log")
mpl.figure(6)
mpl.plot(variable, d_c, linewidth=0.75, label=seccondvar)
mpl.legend(fontsize=15)
mpl.xlabel(label1, fontsize=15)
mpl.ylabel(r"$\Delta_{c}$", rotation=0, labelpad=20, fontsize=15)
mpl.xscale("log")
mpl.figure(7)
mpl.plot(variable,
abs(np.array(acoll) - 1),
linewidth=0.75,
label=seccondvar)
mpl.legend()
mpl.xlabel(label1)
mpl.ylabel(r"$|a_c - 1|$", rotation=0, labelpad=20)
mpl.xscale("log")
mpl.figure(8)
mpl.plot(variable, delta_c, linewidth=0.75, label=seccondvar)
mpl.legend()
mpl.xlabel(label1)
mpl.ylabel(r"$\delta_i$", rotation=0, labelpad=20)
mpl.xscale("log")
mpl.yscale("log")
mpl.figure(1)
mpl.axhline(y=295.6181,
color="red",
linestyle="--",
linewidth=0.75,
label=r"$\Lambda$CDM")
mpl.legend()
mpl.title(
r"$z_{\mathrm{ssb}} = %.1f$, $L = %.1f \mathrm{h}^{-1} \mathrm{Mpc}$"
% (z_ssb, L))
mpl.savefig("Figures\Symmetron\Betadvir.pdf", bbox_inches="tight")
mpl.clf()
mpl.figure(2)
mpl.axhline(y=7.0985,
color="red",
linestyle="--",
linewidth=0.75,
label=r"$\Lambda$CDM")
mpl.legend()
mpl.title(
r"$z_{\mathrm{ssb}} = %.1f$, $L = %.1f \mathrm{h}^{-1}\mathrm{Mpc}$"
% (z_ssb, L))
mpl.savefig("Figures\Symmetron\Betadta.pdf", bbox_inches="tight")
mpl.clf()
mpl.figure(3)
mpl.axhline(y=0.481722,
color="red",
linestyle="--",
linewidth=0.75,
label=r"$\Lambda$CDM")
mpl.legend()
mpl.title(
r"$z_{\mathrm{ssb}} = %.1f$, $L = %.1f \mathrm{h}^{-1}\mathrm{Mpc} $"
% (z_ssb, L))
mpl.savefig("Figures\Symmetron\Betarvirrta.pdf", bbox_inches="tight")
mpl.clf()
mpl.figure(4)
mpl.axhline(y=1.6697,
color="red",
linestyle="--",
linewidth=0.75,
label=r"$\Lambda$CDM")
mpl.legend()
mpl.title(
r"$z_{\mathrm{ssb}} = %.1f$, $L = %.1f \mathrm{h}^{-1}\mathrm{Mpc}$"
% (z_ssb, L))
mpl.savefig("Figures\Symmetron\Betaavirata.pdf", bbox_inches="tight")
mpl.clf()
mpl.figure(5)
mpl.axhline(y=0.918889,
color="red",
linestyle="--",
linewidth=0.75,
label=r"$\Lambda$CDM")
mpl.legend()
mpl.title(
r"$z_{\mathrm{ssb}} = %.1f$, $L = %.1f\mathrm{h}^{-1} \mathrm{Mpc} $"
% (z_ssb, L))
mpl.savefig("Figures\Symmetron\Betaavir.pdf", bbox_inches="tight")
mpl.clf()
mpl.figure(6)
mpl.axhline(y=381.0018577,
color="red",
linestyle="--",
linewidth=0.75,
label=r"$\Lambda$CDM")
mpl.legend()
mpl.title(
r"$z_{\mathrm{ssb}} = %.1f$, $L = %.1f \mathrm{h}^{-1}\mathrm{Mpc} $"
% (z_ssb, L))
mpl.savefig("Figures\Symmetron\Betadelta_c.pdf", bbox_inches="tight")
mpl.clf()
mpl.figure(7)
#mpl.axhline(y = 1, color = "red", linestyle = "--", linewidth = 0.75, label = r"$\Lambda$CDM")
mpl.legend()
mpl.title(
r"$z_{\mathrm{ssb}} = %.1f$, $L = %.1f \mathrm{h}^{-1}\mathrm{Mpc} $"
% (z_ssb, L))
mpl.savefig("Figures\Symmetron\Betaa_c.pdf", bbox_inches="tight")
mpl.clf()
mpl.figure(8)
mpl.axhline(y=0.00059037,
color="red",
linestyle="--",
linewidth=0.75,
label=r"$\Lambda$CDM")
mpl.legend()
mpl.title(
r"$z_{\mathrm{ssb}} = %.1f$, $L = %.1f \mathrm{h}^{-1} \mathrm{Mpc} $"
% (z_ssb, L))
mpl.savefig("Figures\Symmetron\Betadelta_i.pdf", bbox_inches="tight")
mpl.clf()
return
if plot_best_fits:
pylab.rcParams["figure.figsize"] = 16, 6
copy1 = sp.copy()
argxmin = np.argmin(np.abs(sp.xarr.value - xminc))
argxmax = np.argmin(np.abs(sp.xarr.value - xmaxc))
copy1.crop(argxmin, argxmax)
pylab.figure()
pylab.ylim(ymin=0, ymax=1.1 * copy1.data.max())
pylab.xlim(xmin=xminc - 100.0, xmax=xmaxc + 100.0)
#pylab.ylabel(r'$10^{'+str(mag_order-8)+'}$ erg s$^{-1}$ $\AA^{-1}$')
#pylab.xlabel(r'$\AA$')
pylab.yscale('linear')
try:
pylab.plot(sp.xarr,
sp.data,
'k',
label=' fluxcenter rat=' + str(np.round(rat_flux, 2)) +
' fluxwingfar rat=' + str(np.round(rat_fluxww, 2)) +
' fluxwing rat=' + str(np.round(rat_fluxw, 2)) +
' SN=' + str(np.round(SN, 2)) + 'chi2=' +
str(np.round(chi2, 2)) + ' abs rat=' +
str(np.round(rat_absw, 2)) + ' exccess rat=' +
str(np.round(rat_res, 2)) + ' abs rat core=' +
str(np.round(rat_absc, 2)))
except:
pylab.plot(sp.xarr, sp.data, 'k')
def compute_perf(n_vect=None,
p_vect=None,
dur_vect=None,
plot=True,
save=True,
device=None):
n_cpus = int(os.cpu_count())
n_gpus = int(torch.cuda.device_count() * device == 'cuda')
# Batch size
if n_vect is None:
n_vect = [1, 10, 20, 40]
if len(n_vect):
n_vect = np.array(n_vect, dtype=int)
T = 200
colnames = ['version'] + [str(n) for n in n_vect] + [
'number', 'time_length', 'cpus', 'gpus'
]
fname = os.path.join('tests', 'batch_size_perf.csv')
if os.path.exists(fname):
df = pd.read_csv(fname,
header=0,
index_col=0,
dtype={
'version': str,
'number': int,
'time_length': int,
'cpus': int,
'gpus': int
})
df = df.rename(columns=lambda x: x.strip())
if version in df.version.values:
warnings.warn(f'Version label {version} already exists.')
else:
df = pd.DataFrame(columns=colnames)
df = df.append({'version': version}, ignore_index=True)
number = 200
times = []
tm = TransitModule(torch.linspace(5, 7, T)).to(device)
for n in n_vect:
tm.reset_params()
tm.set_params(**get_pars(n))
tm = tm.to(device)
times += [timeit.timeit(tm, number=number) / number]
df.iloc[-1, range(1, 1 + len(n_vect))] = times
df.loc[df.index[-1], 'cpus'] = int(n_cpus)
df.loc[df.index[-1], 'gpus'] = int(n_gpus)
df.loc[df.index[-1], 'number'] = int(number)
df.loc[df.index[-1], ['time_length']] = int(T)
if save:
df.to_csv(fname)
if plot:
plt.figure()
for i in range(len(df)):
if i < len(df) - 1:
kwargs = {'alpha': 0.7, 'linewidth': 0.7}
else:
kwargs = {'color': 'red', 'linewidth': 1.5}
p = plt.plot(n_vect,
df.iloc[i, range(1, 1 + len(n_vect))],
label=str(df.version.iloc[i]) + '-' +
str(df.index[i]) + f'({device})',
**kwargs)
plt.scatter(n_vect,
df.iloc[i, range(1, 1 + len(n_vect))],
s=7,
c=p[0].get_color())
plt.xlabel('batch size ')
plt.ylabel('exec time [s]')
plt.legend()
if save:
plt.savefig(os.path.join('tests', 'batch_size_perf.png'))
else:
plt.show()
# Transit duration
if p_vect is None:
p_vect = np.arange(2, 20, 7)
if len(p_vect):
p_vect = np.array(p_vect, dtype=int)
number = 50
batch_size = 32
colnames = ['version'] + [str(P) for P in p_vect
] + ["batch_size", "number", 'cpus', 'gpus']
fname = os.path.join('tests', 'transit_dur_perf.csv')
if os.path.exists(fname):
df = pd.read_csv(fname,
header=0,
index_col=0,
dtype={
'version': str,
'cpus': int,
'gpus': int,
'batch_size': int,
'number': int
})
df = df.rename(columns=lambda x: x.strip())
if version in df.version.values:
warnings.warn(f'Version label {version} already exists.')
else:
df = pd.DataFrame(columns=colnames)
df = df.append({'version': version}, ignore_index=True)
times = []
durations = []
tm = TransitModule(torch.linspace(4, 6, 1000)).to(device)
for P in p_vect:
tm.reset_params()
tm.set_params(**get_pars(batch_size))
tm.set_param('P', P)
tm = tm.to(device)
durations += [tm.duration[0]]
times += [timeit.timeit(lambda: tm(), number=number) / number]
df.iloc[df.index[-1], range(1, 1 + len(p_vect))] = times
df.loc[df.index[-1], 'cpus'] = int(n_cpus)
df.loc[df.index[-1], 'gpus'] = int(n_gpus)
df.loc[df.index[-1], 'number'] = int(number)
df.loc[df.index[-1], 'batch_size'] = int(batch_size)
if save:
df.to_csv(fname)
if plot:
plt.figure()
for i in range(len(df)):
if i < len(df) - 1:
kwargs = {'alpha': 0.7, 'linewidth': 0.7}
else:
kwargs = {'color': 'red', 'linewidth': 1.5}
p = plt.plot(p_vect,
df.iloc[i, range(1, 1 + len(p_vect))],
label=str(df.version.iloc[i]) + '-' +
str(df.index[i]) + f'({device})',
**kwargs)
plt.scatter(p_vect,
df.iloc[i, range(1, 1 + len(p_vect))],
s=7,
c=p[0].get_color())
plt.xlabel('Transit duration [d]')
# plt.yscale('log')
plt.ylabel('exec time [s]')
plt.legend()
if save:
plt.savefig(os.path.join('tests', 'transit_dur_perf.png'))
else:
plt.show()
# Time vector length
if dur_vect is None:
dur_vect = np.int_(10**torch.arange(6))
if len(dur_vect):
dur_vect = np.array(dur_vect, dtype=int)
number = 10
batch_size = 4
colnames = ['version'] + [str(int(T)) for T in dur_vect
] + ["batch_size", "number", 'cpus', 'gpus']
fname = os.path.join('tests', 'time_length_perf.csv')
if os.path.exists(fname):
df = pd.read_csv(fname,
header=0,
index_col=0,
dtype={
'version': str,
'cpus': int,
'gpus': int,
'batch_size': int,
'number': int
})
df = df.rename(columns=lambda x: x.strip())
if version in df.version.values:
warnings.warn(f'Version label {version} already exists.')
else:
df = pd.DataFrame(columns=colnames)
df = df.append({'version': version}, ignore_index=True)
times = []
tm = TransitModule()
for T in dur_vect:
tm.reset_time()
tm.set_time(torch.linspace(4, 6, T))
tm.reset_params()
tm.set_params(**get_pars(batch_size))
tm = tm.to(device)
times += [timeit.timeit(lambda: tm(), number=number) / number]
df.iloc[-1, range(1, 1 + len(dur_vect))] = times
df.loc[df.index[-1], 'cpus'] = int(n_cpus)
df.loc[df.index[-1], 'gpus'] = int(n_gpus)
df.loc[df.index[-1], 'number'] = int(number)
df.loc[df.index[-1], 'batch_size'] = int(batch_size)
if save:
df.to_csv(fname)
if plot:
plt.figure()
for i in range(len(df)):
if i < len(df) - 1:
kwargs = {'alpha': 0.7, 'linewidth': 0.7}
else:
kwargs = {'color': 'red', 'linewidth': 1.5}
p = plt.plot(dur_vect,
df.iloc[i, range(1, 1 + len(dur_vect))],
label=str(df.version.iloc[i]) + '-' +
str(df.index[i]) + f'({device})',
**kwargs)
plt.scatter(dur_vect,
df.iloc[i, range(1, 1 + len(dur_vect))],
s=7,
c=p[0].get_color())
plt.legend()
plt.yscale('log')
plt.xscale('log')
plt.ylabel('exec time [s]')
plt.xlabel('Time vector length [time steps]')
if save:
plt.savefig(os.path.join('tests', 'time_length_perf.png'))
else:
plt.show()
def plot_mse_beta_compare_start_vectors(BASE_PATH_RANDOM, BASE_PATH_WARM,
INPUT_DATA_X, INPUT_DATA_y):
cv = os.listdir(BASE_PATH_WARM)
for cv_index in range(len(cv)):
print("CV : %s" % cv[cv_index])
PATH_WARM = os.path.join(BASE_PATH_WARM, cv[cv_index], "all")
PATH_RANDOM = os.path.join(BASE_PATH_RANDOM, cv[cv_index], "all")
params = glob.glob(os.path.join(PATH_WARM, "0*"))
for p in params:
p = os.path.basename(p)
print(p)
snap_path_warm = os.path.join(PATH_WARM, p,
"conesta_ite_snapshots")
conesta_ite_warm = sorted(os.listdir(snap_path_warm))
nb_conesta_warm = len(conesta_ite_warm)
snap_path_random = os.path.join(PATH_RANDOM, p,
"conesta_ite_snapshots")
conesta_ite_random = sorted(os.listdir(snap_path_random))
nb_conesta_random = len(conesta_ite_random)
ite_final_warm = np.load(
os.path.join(snap_path_warm, conesta_ite_warm[-1]))
beta_star_warm = ite_final_warm["beta"]
gap_warm = ite_final_warm["gap"]
mse_warm = np.zeros((nb_conesta_warm))
i = 0
for ite in conesta_ite_warm:
path = os.path.join(snap_path_warm, ite)
ite_warm = np.load(path)
mse_warm[i] = sklearn.metrics.mean_squared_error(
ite_warm["beta"][:, 0], beta_star_warm[:, 0])
i = i + 1
ite_final_random = np.load(
os.path.join(snap_path_random, conesta_ite_random[-1]))
beta_star_random = ite_final_random["beta"]
gap_random = ite_final_random["gap"]
mse_random = np.zeros((nb_conesta_random))
i = 0
for ite in conesta_ite_random:
path = os.path.join(snap_path_random, ite)
ite_random = np.load(path)
mse_random[i] = sklearn.metrics.mean_squared_error(
ite_random["beta"][:, 0], beta_star_random[:, 0])
i = i + 1
pdf_path = os.path.join(BASE_PATH_WARM, cv[cv_index], "all", p,
"start_vector_effect_on_gap.pdf")
pdf = PdfPages(pdf_path)
fig = plt.figure(figsize=(11.69, 8.27))
plt.plot(gap_random, label="Random start")
plt.plot(gap_warm, label="Warm start: Beta from all/all")
plt.xscale('log')
plt.yscale('log')
plt.xlabel("iterations")
plt.ylabel(r"$gap$")
plt.legend(prop={'size': 15})
plt.title(p)
pdf.savefig()
plt.close(fig)
pdf.close()
def magentar_plot():
xmm=ascii.read('data/xmm_SED_paper.dat',names=['energy','err_energy','flux','err_flux'])
integral=ascii.read('data/integral_SED_paper.dat',names=['energy','err_energy','flux','err_flux'])
comptel=ascii.read('data/comptel_SED_paper.dat',names=['energy','err_energy','flux','err_flux'])
lat={'energy_min':[1e5,1e5,1e6],'energy_max':[1e7,1e6,1e7],'flux':[1.6e-4,2.1e-4,8.13e-5]}
#magic={'energy_min':[200e3,775e3],'energy_max':[774e3,3000e3],'flux':[6.29e-5,8.99e-5]}
models=ascii.read('data/magnetar_4U0142+614.dat',names=['energy','flux'])
fig=plot.figure()
xrange=[1e-4,1e5]
yrange=[5e-13,3e-10]
s=1e-8
plot.fill_between([0.2,10e3],[yrange[1],yrange[1]],[yrange[0],yrange[0]],facecolor='yellow',interpolate=True,color='yellow',alpha=0.5)
plot.annotate('AMEGO',xy=(1.5,1.7e-10),xycoords='data',fontsize=26,color='black')
plot.errorbar(xmm['energy']*1e-3,xmm['flux']*s,xerr=xmm['err_energy']*1e-3,yerr=xmm['err_flux']*s,capsize=0, fmt='.',color='blue')
plot.errorbar(integral['energy']*1e-3,integral['flux']*s,xerr=integral['err_energy']*1e-3,yerr=integral['err_flux']*s,capsize=0, fmt='.',color='blue')
plot.errorbar(comptel['energy']*1e-3,comptel['flux']*s,xerr=comptel['err_energy']*1e-3,yerr=0.5*comptel['flux']*s,capsize=0, fmt='.',color='blue',uplims=True)
m=np.array(10**((np.log10(lat['energy_min'])+np.log10(lat['energy_max']))/2))*1e-3
lowe=m-np.array(lat['energy_min'])*1e-3
highe=np.array(lat['energy_max'])*1e-3-m
plot.errorbar(m,np.array(lat['flux'])*s,xerr=[lowe,highe],yerr=0.3*np.array(lat['flux'])*s,capsize=0, fmt='.',color='blue',uplims=True)
# m=np.array(10**((np.log10(magic['energy_min'])+np.log10(magic['energy_max']))/2))
# lowe=m-np.array(magic['energy_min'])
# highe=np.array(magic['energy_max'])-m
# #plot.errorbar(m,np.array(magic['flux'])*s,xerr=[lowe,highe],yerr=0.2*np.array(magic['flux'])*s,capsize=0, fmt='.',color='blue',uplims=True)
erg2kev=624151.*1e3
color=['red','magenta','black','black']
lines=['','','','r--']
num=[0,18,38,67,len(models['energy'])]
for i in range(0,4):
n0=num[i]
n1=num[i+1]
ind=np.arange(n0,n1)
x=np.array(models['energy'])*1e-3
y=np.array(models['flux'])/erg2kev
# n2=(n1-n0)/2+n0
w=np.where(y[ind]-max(y[ind]) == 0)
n2=ind[w[0][0]]
x0=loginterpol(y[n0:n2],x[n0:n2],yrange[0])
x1=loginterpol(y[n2:n1],x[n2:n1],yrange[0])
y=np.append(np.append(yrange[0],y[ind]),yrange[0])
x=np.append(np.append(x0,x[ind]),x1)
# y=np.append(yrange[0],y)
# x=np.append(x0,x)
plot.plot(x,y,lines[i],color=color[i])
#y=np.append(np.append(yrange[0],models['flux'][ind]/erg2kev),yrange[0])
#x=loginterpol(models['flux'][ind[0:9]]/erg2kev,models['energy'][ind[0:9]*1e-3,y)
#plot.plot(x,y,color='red')
#print x[0],x[19]
#print y[0],y[19]
# ind=np.arange(18,38)
# plot.plot(models['energy'][ind]*1e-3,models['flux'][ind]/erg2kev,color='magenta')
# ind=np.arange(38,67)
# plot.plot(models['energy'][ind]*1e-3,models['flux'][ind]/erg2kev,color='black')
# ind=np.arange(67,len(models['flux']))
# plot.plot(models['energy'][ind]*1e-3,models['flux'][ind]/erg2kev,'r--',color='black')
plot.xscale('log')
plot.yscale('log')
plot.ylim(yrange)
plot.xlim(xrange)
plot.xlabel(r'Energy (MeV)')
plot.ylabel(r'Energy$^2 \times $ Flux (Energy) (erg cm$^{-2}$ s$^{-1}$)')
plot.show()
def execute(self, dictFileName):
print('Starting {0:s} tool'.format(self.__toolName__), flush=True)
print("Start time {}\n".format(
strftime("%Y-%m-%d %H:%M:%S", localtime())), flush=True)
lines = self.read_dict(dictFileName)
empty, initialDir, description = \
self.check_a_parameter('initialDir', lines)
empty, poreDir, description = \
self.check_a_parameter('poreDir', lines)
empty, crystalDir, description = \
self.check_a_parameter('crystalDir', lines)
empty, outputDir, description = \
self.check_a_parameter('outputDir', lines)
# there should be only one file in initialDir
ini_dir_files = sorted([f for f in os.listdir(initialDir)
if os.path.isfile(os.path.join(initialDir, f))
and '.tif' in f])
img0 = io.imread(os.path.join(initialDir,ini_dir_files[0]),
plugin='tifffile')
im0_bin = img0.astype(bool).astype(np.int8)
pore_volume = im0_bin.shape[0] * im0_bin.shape[1] * im0_bin.shape[2] \
- np.sum(im0_bin)
print("Pore volume: {}".format(pore_volume))
# get data file names
crystal_files = sorted([f for f in os.listdir(crystalDir)
if os.path.isfile(os.path.join(crystalDir, f))])
# get file names from pore dir
pore_files = sorted([f for f in os.listdir(poreDir)
if os.path.isfile(os.path.join(poreDir, f))])
KGG = self.N11(im0_bin, self.KER)
KGG[KGG == 6] = 0
# only the surface pixels are not 0
KGG = KGG.astype(bool).astype(np.int8)
# number of surface voxels
Nsv = np.sum(KGG)
print("Number of surface voxels: {}".format(Nsv))
print("ini file time: {} min".
format( self.extract_time_from_filename(ini_dir_files[0]) ))
colors = ['g-', 'b-', 'r-', 'k-']
colors1 = ['go', 'bo', 'ro', 'ko']
solid_bin = deepcopy(im0_bin)
previous_crystals = np.zeros(im0_bin.shape)
ini_time = 0
#axX1=[]
#axY1=[]
j = 0
for i, file in enumerate(crystal_files):
file_path = os.path.join(crystalDir, file)
img = io.imread(file_path, plugin='tifffile')
# crystals are 1, rest - 0
im_bin = img.astype(bool).astype(np.int8)
# read pore files
pore_path = os.path.join(poreDir, pore_files[i])
pore_img = io.imread(pore_path, plugin='tifffile')
# number of crystal voxels
#nCryst = np.sum(im_bin)
fnm = file.split("_")
time = 0
for ww in fnm:
if "min" in ww:
time = int(ww[:-3])
print("{} time: {} min".format(i, time))
#axX, axY = self.plot_pore_vs_h(pore_img)
#axX, axY = self.plot_nCryst_per_surf(KGG, im_bin)
axX, axY = self.plot_Ncryst_vsPoreSize(im_bin, pore_img)
#previous_crystals = np.multiply(previous_crystals, im_bin)
#add_cryst = np.subtract(im_bin, previous_crystals)
#add_cryst[add_cryst < 0] = 0
#axX, axY = self.plot_nucleation(im0_bin, im_bin,
# previous_crystals, add_cryst)
#axY = self.plot_clusters1(im0_bin, im_bin,
# previous_crystals, add_cryst)
#axX, axY = self.cluster_hist(im_bin)
#previous_crystals = im_bin
dtime = time - ini_time
#axY = axY / dtime
ini_time = time
#axX1.append(time)
#axY1.append(axY)
#YY1 = np.polyfit(axX, axY, 3)
#pf = np.poly1d(YY1)
#pfX = np.linspace(axX[0], axX[-1], 100)
#plt.plot(pfX, pf(pfX), colors[i])
if i%3==0:
print("++++++++++++++++++++++++++++++++++++++++++++++++++++++++")
plt.plot(axX, axY, colors1[j], label='t={}'.format(time))
j=j+1
#solid_bin = np.add(im0_bin, im_bin)
#break
#axX1 = np.asarray(axX1)
#axY1 = np.asarray(axY1)
#plt.plot(axX1, axY1, colors1[i])
plt.xscale('log')
plt.yscale('log')
plt.tight_layout()
plt.show()
#res_file_name = "Ncr_perSurfperdt_vsH.png"
#res_file_name = "meanPoreSize_vs_H.png"
#res_file_name = "Surf_vsH.png"
res_file_name = "histPoreSize.png"
#res_file_name = "Ncr_perPoreVol_vsSize.png"
res_file_path = os.path.join(outputDir,res_file_name)
#plt.savefig(res_file_path, format='png', dpi=300)
#plt.savefig(res_file_name, format='pdf', dpi=300)
print("End time {}".format(
strftime("%Y-%m-%d %H:%M:%S", localtime())), flush=True)
return True
def longStats(af = 4, filter=None):
dataDirectory = "/data/routeviews/archive.routeviews.org/route-views.linx/bgpdata/"
space = 1
yearRange = range(2004, 2017)
#monthRange = range(1,13)
monthRange = [6] #range(1,13)
day = 15
dateRange = []
tier1 = {"3356":[], "1299":[], "174":[] ,"2914":[],"3257":[]}#, "6453":[], "3491":[], "701":[], "1239":[], "6762":[]}
# Find the first RIB files for each year
ribFiles = []
for ye in yearRange:
for month in monthRange:
ribs = glob.glob(dataDirectory+"{ye}.{mo:02d}/RIBS/rib.{ye}{mo:02d}{da:02d}.*.bz2".format(ye=ye, mo=month, da=day))
ribs.sort()
if len(ribs) < 1:
continue
ribFiles.append(((ye,month,day),ribs[0]))
dateRange.append(datetime.datetime(ye,month,day))
outDir = "../results/longStats_space%s_ipv%s/" % (space, af)
try:
os.makedirs(os.path.dirname(outDir))
except OSError as exc: # Guard against race condition
if exc.errno != errno.EEXIST:
raise
plt.figure()
ccmap = mpl.cm.get_cmap('copper_r')
# Using contourf to provide my colorbar info, then clearing the figure
Z = [[0,0],[0,0]]
CS3 = plt.contourf(Z, yearRange, cmap=ccmap)
plt.clf()
for i, (date, ribFile) in enumerate(ribFiles):
print date
if filter is None:
centralityFile = outDir+"/%s%02d%02d_af%s.pickle" % (date[0], date[1], date[2], af)
fList = None
else:
centralityFile = outDir+"/%s%02d%02d_AS%s_af%s.pickle" % (date[0], date[1], date[2],filter, af)
fList = [filter]
if not os.path.exists(centralityFile):
rtree, _ = ashash.readrib(ribFile, space, af, filterAS=fList)
asAggProb, asProb, _ = ashash.computeCentrality(rtree.search_exact("0.0.0.0/0").data, space, filterAS=filter)
pickle.dump((asAggProb, asProb), open(centralityFile, "wb"))
else:
asAggProb, asProb = pickle.load(open(centralityFile,"rb"))
if asAggProb is None or len(asAggProb) < 1:
continue
if filter is None and af==4:
for k,v in tier1.iteritems():
v.append(asAggProb[str(k)])
if not filter is None:
del asAggProb[str(filter)]
sortedAS = sorted(asAggProb, key=lambda k: asAggProb[k], reverse=True)
i=0
while i<len(sortedAS) and asAggProb[sortedAS[i]]>.05:
print "%s: %s" % (sortedAS[i], asAggProb[sortedAS[i]])
i+=1
eccdf(asAggProb.values(), lw=1.3, label=date[0],c=ccmap(i/float(len(ribFiles)) ) )
# print date
# maxKey = max(asAggProb, key=asAggProb.get)
# print "AS%s = %s" % (maxKey, asAggProb[maxKey])
plt.grid(True)
#plt.legend(loc="right")
plt.colorbar(CS3)
plt.xscale("log")
plt.yscale("log")
#plt.yscale("log")
# plt.xlim([10**-8, 10**-2])
if filter is None:
plt.xlim([10**-7, 1.1])
else:
plt.xlim([10**-7, 1.1])
# plt.ylim([10**-3, 1.1])
plt.xlabel("AS hegemony")
plt.ylabel("CCDF")
plt.tight_layout()
if filter is None:
plt.title("IPv%s" % af)
plt.savefig(centralityFile.rpartition("/")[0]+"/hegemonyLongitudinal_af%s.eps" % af)
else:
plt.title("AS%s (IPv%s)" % (filter, af))
plt.savefig(centralityFile.rpartition("/")[0]+"/hegemonyLongitudinal_AS%s_af%s.eps" % (filter, af))
if filter is None and af==4:
fig = plt.figure(figsize=(10,3))
for k,v in tier1.iteritems():
plt.plot(dateRange, v, label="AS"+k)
plt.ylim([0,0.3])
plt.grid(True)
plt.ylabel("AS hegemony")
plt.xlabel("Time")
plt.legend(loc="center", bbox_to_anchor=(0.5, 1), ncol=len(tier1))
plt.tight_layout()
plt.savefig(centralityFile.rpartition("/")[0]+"/tier1.eps")
def plot_mse_vs_gap_all_params(BASE_PATH, INPUT_DATA_X, INPUT_DATA_y):
###PLot correlationbetween beta stability and precision
X = np.load(INPUT_DATA_X)
y = np.load(INPUT_DATA_y)
cv = glob.glob(BASE_PATH)
for i in range(len(cv)):
print("CV : %s" % cv[i])
PATH = cv[i]
params = glob.glob(os.path.join(PATH, "0*"))
for p in params:
p = os.path.basename(p)
print(p)
print("CV number : %s" % cv)
pdf_path = os.path.join(PATH, "mse_vs_gap.pdf")
pdf = PdfPages(pdf_path)
paired_pal = sns.color_palette("Paired", 10)
colors = {
("0.01_0.08_0.72_0.2"): paired_pal[0],
("0.1_0.08_0.72_0.2"): paired_pal[1],
("0.01_0.18_0.02_0.8"): paired_pal[2],
("0.1_0.18_0.02_0.8"): paired_pal[3],
("0.01_0.72_0.08_0.2"): paired_pal[4],
("0.1_0.72_0.08_0.2"): paired_pal[5],
("0.01_0.02_0.18_0.8"): paired_pal[6],
("0.1_0.02_0.18_0.8"): paired_pal[7]
}
fig = plt.figure(figsize=(11.69, 8.27))
params = glob.glob(os.path.join(BASE_PATH, "0*"))
for p in params:
p = os.path.basename(p)
print(p)
snap_path = os.path.join(PATH, p, "conesta_ite_snapshots")
conesta_ite = sorted(os.listdir(snap_path))
nb_conesta = len(conesta_ite)
if nb_conesta != 0:
ite_final = np.load(
os.path.join(snap_path, conesta_ite[-1]))
beta_star = ite_final["beta"]
mse = np.zeros((nb_conesta))
i = 0
for ite in conesta_ite:
path = os.path.join(snap_path, ite)
ite = np.load(path)
mse[i] = sklearn.metrics.mean_squared_error(
ite["beta"][:, 0], beta_star[:, 0])
i = i + 1
gap = np.zeros((nb_conesta))
for i in range(len(conesta_ite)):
fista_number = ite['continuation_ite_nb'][i]
gap[i] = ite["gap"][fista_number - 1]
plt.plot(mse, np.log10(gap), color=colors[(p)], label=p)
plt.xlabel(r"$mse(\beta^{k}$ - $\beta^{*})$ ")
plt.ylabel(r"precision")
plt.yscale('log')
plt.legend(prop={'size': 15})
plt.title("3D voxel-based GM maps - 257595 features")
fig.tight_layout()
pdf.savefig()
plt.close(fig)
pdf.close()
def plot_variance_reduction_geometric_brownian_motion(savefig=False,
plot_from_json=True):
methods = ['euler_maruyama', 'milstein']
if plot_from_json:
results = {}
for method in methods:
with open('variance_reduction_{}_scheme.json'.format(method),
"r") as input_file:
results[method] = json.load(input_file)
results[method] = {
k: {float(x): y
for x, y in v.items()}
for k, v in results[method].items()
}
else:
deltas = [2.0**-i for i in range(1, 7)]
inverse_norm = norm.ppf
piecewise_constant = construct_piecewise_constant_approximation(
inverse_norm, n_intervals=1024)
piecewise_linear = construct_symmetric_piecewise_polynomial_approximation(
inverse_norm, n_intervals=16, polynomial_order=1)
piecewise_cubic = construct_symmetric_piecewise_polynomial_approximation(
inverse_norm, n_intervals=16, polynomial_order=3)
approximations = {
'constant': piecewise_constant,
'linear': piecewise_linear,
'cubic': piecewise_cubic,
'rademacher': rademacher_approximation
}
results = {
method:
{term: {}
for term in ['original'] + list(approximations.keys())}
for method in methods
} # Store the values of delta and the associated data.
time_per_level = 2.0
paths_min = 64
for method in results:
for approx_name, approx in approximations.items():
for dt in deltas:
_, elapsed_time_per_path = time_function(
produce_geometric_brownian_motion_paths)(dt, method,
approx)
paths_required = int(time_per_level /
elapsed_time_per_path)
if paths_required < paths_min:
print("More time required for {} and {} with dt={}".
format(method, approx_name, dt))
break
originals, corrections = [
[None for i in range(paths_required)] for j in range(2)
]
for path in range(paths_required):
x_fine_exact, x_coarse_exact, x_fine_approx, x_coarse_approx = produce_geometric_brownian_motion_paths(
dt, method, approx)
originals[path] = x_fine_exact - x_coarse_exact
corrections[path] = min(
(x_fine_exact - x_coarse_exact) -
(x_fine_approx - x_coarse_approx),
(x_fine_exact - x_fine_approx) -
(x_coarse_exact - x_coarse_approx),
sum([
x_fine_exact, -x_coarse_exact, -x_fine_approx,
x_coarse_approx
])
) # might need revising for near machine precision.
originals, corrections = [
[j**2 for j in i] for i in [originals, corrections]
]
for name, values in [['original', originals],
[approx_name, corrections]]:
mean = np.mean(values)
std = np.std(values) / (len(values)**0.5)
[mean, std] = [float(i) for i in [mean, std]]
results[method][name][dt] = [mean, std]
markers = {
'original': 'd',
'constant': 'o',
'linear': 'v',
'cubic': 's',
'rademacher': 'x'
}
deltas = list(list(list(results.items())[0][1].items())[0][1].keys())
for method in results:
plt.clf()
for approx_name in results[method]:
x, y = zip(*results[method][approx_name].items())
y, y_std = list(zip(*y))
y_error = 1 * np.array(y_std)
plt.errorbar(x, y, y_error, None,
'k{}:'.format(markers[approx_name]))
plt.xscale('log', basex=2)
plt.yscale('log', basey=2)
plt.xlabel(r'Fine time increment $\delta^{\mathrm{f}}$')
plt.ylabel('Variance')
y_min_base_2 = 50
plt.ylim(2**-y_min_base_2, 2**-10)
plt.yticks([2**-i for i in range(10, y_min_base_2 + 1, 10)])
plt.xticks(deltas)
if savefig:
plt.savefig('variance_reduction_{}_scheme.pdf'.format(method),
format='pdf',
bbox_inches='tight',
transparent=True)
if not plot_from_json:
with open('variance_reduction_{}_scheme.json'.format(method),
"w") as output_file:
output_file.write(json.dumps(results[method], indent=4))
def plot_fit(self):
'''
Plots the best fit model to the data.
'''
lam = np.linspace( np.min(self.wls) -1000 , np.max(self.wls) + 500, 1000)
plt.clf()
plt.figure(figsize=(8,6))
plt.errorbar(self.wls, self.fluxes, yerr=self.fluxerrs, fmt="o")
for i in range(len(self.wls)):
plt.text(self.wls[i], self.fluxes[i]*1.01, self.bands[i].split(",")[-1], alpha=.4)
if self.model == "BlackBody":
fluxbb = self._model(lam, (self.T, self.R))
plt.plot(lam, fluxbb, "k-", label="BB fit")
plt.title("T: %.1f K R:%.1f R$_{\odot}$ Lumiosity %.1e L$_{\odot}$"%(self.T, self.R, self.L))
elif self.model == "BlackBody_Av":
fluxbb = self._model(lam, (self.T, self.R))
fluxbb_red = self._model_av(lam, (self.T, self.R, self.Av))
plt.plot(lam, fluxbb, "k-", label="BB fit")
plt.plot(lam, fluxbb_red, "red", label="BB fit + reddening")
plt.title("T: %.1f K R:%.1f R$_{\odot}$ Lumiosity %.1e L$_{\odot}$ Av: %.2f"%(self.T, self.R, self.L, self.Av))
elif self.model == "BlackBody2_Av":
fluxbb_red = self._model_av(lam, (self.T, self.R, self.Av))
fluxbb_secondary_red = self._model_av(lam, (self.Tsec, self.Rsec, self.Av))
fluxbb_with_seconday = self._model2_av(lam, (self.T, self.R, self.Av, self.Tsec, self.Rsec))
plt.plot(lam, fluxbb_red, "k-", label="BB1 fit + reddening")
plt.plot(lam, fluxbb_secondary_red, "k--", label="BB2 fit + reddening")
plt.plot(lam, fluxbb_with_seconday, "green", label="BB1 + BB2")
plt.title("T: %.1f K R:%.1f R$_{\odot}$ Lumiosity %.1e L$_{\odot}$ Av: %.2f\n T2: %.1f R2: %.1f"%(self.T, \
self.R, self.L, self.Av, self.Tsec, self.Rsec))
elif self.model == "BlackBody2":
fluxbb_primary = self._model(lam, (self.T, self.R))
fluxbb_secondary = self._model(lam, (self.Tsec, self.Rsec))
fluxbb_with_seconday = self._model2_r(lam, (self.T, self.R, self.Tsec, self.Rsec))
plt.plot(lam, fluxbb_primary, "k-", label="BB1 fit")
plt.plot(lam, fluxbb_secondary, "k--", label="BB2 fit")
plt.plot(lam, fluxbb_with_seconday, "green", label="BB1 + BB2")
plt.title("T: %.1f K R:%.1f R$_{\odot}$ Lumiosity %.1e L$_{\odot}$ T2: %.1f R2: %.1f"%(self.T, \
self.R, self.L, self.Tsec, self.Rsec))
elif self.model == "PowerLaw":
flux = self._model_powerlaw(lam, (self.alpha, self.R, self.Av))
plt.plot(lam, flux, "k-", label="PowerLaw + reddening")
plt.title("$\\alpha$: %.1f Av: %.2f"%(self.alpha, self.Av))
plt.xlabel("Wavelength [$\\AA$]")
plt.ylabel("log Flux")
#plt.ylim(ymin=np.min(self.fluxes) * 0.8)
plt.yscale("log")
plt.legend()
name = self._get_save_path(None, "mcmc_best_fit_model")
plt.savefig(name)
plt.close("all")
# -----------set up units properly------------------#
mag_order=np.int((1)*np.round(np.log10(np.mean(sp.data))))
sp.xarr.units='angstroms'
sp.xarr.xtype = 'wavelength'
sp.units = r'$10^{'+str(mag_order)+'}$ erg s$^{-1}$ $cm^{-1}$'
sp.data /= 10**(mag_order)
#-------------- set up units properly------------#
wlmin=sp.xarr[0]
wlmax=sp.xarr[-1]
pylab.yscale('log')
pylab.xscale('log')
#pylab.xscale('log',subsx=[3,4])
pylab.rcParams["figure.figsize"]=16,6
sp.plotter(figure=1,xmin=wlmin,xmax=wlmax,ymin=1.1*sp.data.min(),ymax=3.5*sp.data.max())
pylab.close()
#-------------continuous subtraction-------------------#
if w=="UV":
continuous,wlmin_UV=continuous_substraction( i, sp, mag_order,UV_limits,w)
backup=sp.copy()
sp.data=sp.data - continuous
balmer_template,balmer_tot, index=balmer_normalization(sp,balmer_cont_template,balmer_lines_template)
sp.data=backup.data
def plot_log(targets):
x = np.linspace(0, len(targets), len(targets))
plt.plot(x, targets)
plt.yscale('symlog')
plt.show()
def compare_start_vectors(BASE_PATH_RANDOM, BASE_PATH_WARM, INPUT_DATA_X,
INPUT_DATA_y):
for cv in range(5):
params = glob.glob(os.path.join(BASE_PATH_RANDOM, "all/all/*"))
for p in params:
p = os.path.basename(p)
print(p)
pdf_path = "/neurospin/brainomics/2017_parsimony_settings/warm_restart/NUSDAST_30yo/VBM/no_covariates/warm_restart/influence_of_start_vector/cv0%s" % cv
os.makedirs(pdf_path, exist_ok=True)
pdf = PdfPages(os.path.join(pdf_path, "%s.pdf" % p))
fig = plt.figure(figsize=(11.69, 8.27))
snap_path_random = os.path.join(BASE_PATH_RANDOM, "cv0%s/all" % cv,
p, "conesta_ite_snapshots/")
conesta_ite_random = sorted(os.listdir(snap_path_random))
nb_conesta_random = len(conesta_ite_random)
ite_final_random = np.load(
os.path.join(snap_path_random, conesta_ite_random[-1]))
beta_star_random = ite_final_random["beta"]
gap_random = ite_final_random["gap"]
mse_random = np.zeros((nb_conesta_random))
i = 0
for ite in conesta_ite_random:
path = os.path.join(snap_path_random, ite)
ite_random = np.load(path)
mse_random[i] = sklearn.metrics.mean_squared_error(
ite_random["beta"][:, 0], beta_star_random[:, 0])
i = i + 1
plt.plot(gap_random, label="Random start")
for warm_num in range(5):
PATH_WARM = BASE_PATH_WARM + "cv0%s_all_as_start_vector/model_selectionCV/cv0%s/all" % (
warm_num, cv)
snap_path_warm = os.path.join(PATH_WARM, p,
"conesta_ite_snapshots")
conesta_ite_warm = sorted(os.listdir(snap_path_warm))
nb_conesta_warm = len(conesta_ite_warm)
ite_final_warm = np.load(
os.path.join(snap_path_warm, conesta_ite_warm[-1]))
beta_star_warm = ite_final_warm["beta"]
gap_warm = ite_final_warm["gap"]
mse_warm = np.zeros((nb_conesta_warm))
i = 0
for ite in conesta_ite_warm:
path = os.path.join(snap_path_warm, ite)
ite_warm = np.load(path)
mse_warm[i] = sklearn.metrics.mean_squared_error(
ite_warm["beta"][:, 0], beta_star_warm[:, 0])
i = i + 1
plt.plot(gap_warm,
label="Warm start: Beta from cv0%s/all" % warm_num)
plt.xscale('log')
plt.yscale('log')
plt.xlabel("iterations")
plt.ylabel(r"$gap$")
plt.legend(prop={'size': 15})
plt.title(p)
plt.tight_layout()
pdf.savefig()
plt.close(fig)
pdf.close()
Omega_m0_max = float(sys.argv[2])
n=11
Omega_m0 = np.linspace(Omega_m0_min, Omega_m0_max, n)
if len(sys.argv) == 3:
Omega_K = 1 - Omega_m0 - Lambda
else:
Omega_K = np.zeros(n)
for j in range(n):
Omega_K[j] = float(sys.argv[3])
omega = Omega_K + Omega_m0 + Lambda
s0 = 0.5
#result = odeint(s, s0, z, args=(Lambda, Omega_m0, Omega_K))
for i in range(len(Omega_m0)):
sofa = odeint(s_a, s0, a, args = (Omega_m0[i], Omega_K[i], Lambda, a0))
mpl.plot(a, sofa, linewidth = 0.75, label = r"s, $\Omega_{m0} =$ %.1f" % Omega_m0[i])
mpl.xlabel("a(t)")
mpl.ylabel("s(a)")
mpl.legend()
mpl.yscale("log")
mpl.xscale("log")
mpl.show()
def makeDailyChannelOffsetSignal( ):
from functions.TAfunctions import SMA, MoveMax, jumpTheChannelTest
import functions.allstats
from functions.UpdateSymbols_inHDF5 import *
from functions.GetParams import GetParams
file4path = os.path.join( os.getcwd(), "pyTAAAweb_DailyChannelOffsetSignal_status.params" )
figure4path = os.path.join( os.getcwd(), "pyTAAA_web", "PyTAAA_DailyChannelOffsetSignalV.png" )
symbol_directory = os.path.join( os.getcwd(), "symbols" )
symbol_file = "Naz100_Symbols.txt"
symbols_file = os.path.join( symbol_directory, symbol_file )
adjClose, symbols, datearray, _, _ = loadQuotes_fromHDF( symbols_file )
###
### get last date already processed
###
_dates = []
avgPctChannel = []
numAboveBelowChannel = []
try:
with open( file4path, "r" ) as f:
# get number of lines in file
lines = f.read().split("\n")
numlines = len (lines)
for i in range(numlines):
statusline = lines[i]
statusline_list = statusline.split(" ")
statusline_list = filter(None, statusline_list)
if len( statusline_list ) == 3:
_dates.append( datetime.datetime.strptime( statusline_list[0], '%Y-%m-%d') )
avgPctChannel.append( float(statusline_list[1].split('%')[0])/100. )
numAboveBelowChannel.append( float(statusline_list[2]) )
except:
print " Error: unable to read updates from pyTAAAweb_numberUptrendingStocks_status.params"
print ""
#print "_dates = ", _dates
last_date = _dates[-1].date()
print " ...inside makeDailyChannelOffsetSignal... last_date = ", last_date
# parameters for signal
params = GetParams()
minperiod = params['minperiod']
maxperiod = params['maxperiod']
incperiod = params['incperiod']
numdaysinfit = params['numdaysinfit']
offset = params['offset']
print "minperiod,maxperiod,incperiod,numdaysinfit,offset = ", minperiod,maxperiod,incperiod,numdaysinfit,offset
# process for each date
print "\n ... inside makeDailyChannelOffsetSignal ..."
dailyChannelOffsetSignal = np.zeros( adjClose.shape[1], 'float' )
dailyCountDowntrendChannelOffsetSignal = np.zeros( adjClose.shape[1], 'float' )
#for idate in range(numdaysinfit+incperiod,adjClose.shape[1])
for idate in range(adjClose.shape[1]):
if datearray[idate] >= last_date :
#if datearray[idate] > datetime.date(1992,1,1) :
#if datearray[idate] > datetime.date(1992,1,1) :
if idate%10 == 0:
print " ...idate, datearray[idate] = ", idate, datearray[idate]
# process all symbols
numberDowntrendSymbols = 0
dailyChannelPct = []
##print " ... symbols = ", symbols
floatChannelGainsLosses = []
floatStdevsAboveChannel = []
for i, symbol in enumerate(symbols):
#print " ... symbol = ", symbol
quotes = adjClose[i,idate-numdaysinfit-offset-1:idate].copy()
channelGainLoss, numStdDevs, pctChannel = \
recentTrendAndStdDevs( \
quotes, \
datearray,\
minperiod=minperiod,\
maxperiod=maxperiod,\
incperiod=incperiod,\
numdaysinfit=numdaysinfit,\
offset=offset)
floatChannelGainsLosses.append(channelGainLoss)
floatStdevsAboveChannel.append(numStdDevs)
floatChannelGainsLosses = np.array(floatChannelGainsLosses)
floatChannelGainsLosses[np.isinf(floatChannelGainsLosses)] = -999.
floatChannelGainsLosses[np.isneginf(floatChannelGainsLosses)] = -999.
floatChannelGainsLosses[np.isnan(floatChannelGainsLosses)] = -999.
floatChannelGainsLosses = floatChannelGainsLosses[floatChannelGainsLosses != -999.]
floatStdevsAboveChannel = np.array(floatStdevsAboveChannel)
floatStdevsAboveChannel[np.isinf(floatStdevsAboveChannel)] = -999.
floatStdevsAboveChannel[np.isneginf(floatStdevsAboveChannel)] = -999.
floatStdevsAboveChannel[np.isnan(floatStdevsAboveChannel)] = -999.
floatStdevsAboveChannel = floatStdevsAboveChannel[floatStdevsAboveChannel != -999.]
##print "floatChannelGainsLosses.shape = ", floatChannelGainsLosses.shape
trimmeanGains = np.mean(floatChannelGainsLosses[np.logical_and(\
floatChannelGainsLosses>np.percentile(floatChannelGainsLosses,5),\
floatChannelGainsLosses<np.percentile(floatChannelGainsLosses,95)\
)])
trimmeanStdevsAboveChannel = np.mean(floatStdevsAboveChannel[np.logical_and(\
floatStdevsAboveChannel>np.percentile(floatStdevsAboveChannel,5),\
floatStdevsAboveChannel<np.percentile(floatStdevsAboveChannel,95)\
)])
#print "idate= ",idate,str(datearray[idate])
textmessage2 = ''
with open( file4path, "a" ) as ff:
textmessage2 = "\n"+str(datearray[idate])+" "+\
format(trimmeanGains,"8.2%")+" "+\
format(trimmeanStdevsAboveChannel,"7.1f")
ff.write(textmessage2)
print "textmessage2 = ", textmessage2
#print "idate= ",idate, str(datearray[idate])
##########################################
# make plot
##########################################
###
### make a combined plot
### 1. get percent of uptrending stocks
###
_dates = []
avgPctChannel = []
numAboveBelowChannel = []
try:
with open( file4path, "r" ) as f:
# get number of lines in file
lines = f.read().split("\n")
numlines = len (lines)
for i in range(numlines):
statusline = lines[i]
statusline_list = statusline.split(" ")
statusline_list = filter(None, statusline_list)
if len( statusline_list ) == 3:
_dates.append( datetime.datetime.strptime( statusline_list[0], '%Y-%m-%d') )
avgPctChannel.append( float(statusline_list[1].split('%')[0])/100. )
numAboveBelowChannel.append( float(statusline_list[2]) )
except:
print " Error: unable to read updates from pyTAAAweb_numberUptrendingStocks_status.params"
print ""
_dates = np.array(_dates)
avgPctChannel = np.array(avgPctChannel)
numAboveBelowChannel = np.array(numAboveBelowChannel)
print " avgPctChannel min, mean, max = ", avgPctChannel.min(),avgPctChannel.mean(),avgPctChannel.max()
print "\n\n numAboveBelowChannel = ", numAboveBelowChannel
print " numAboveBelowChannel min, mean, max = ", numAboveBelowChannel.min(),numAboveBelowChannel.mean(),numAboveBelowChannel.max()
plt.figure(4,figsize=(9,7))
plt.clf()
plt.grid(True)
numDaysToPlot = 252*3
plt.plot( _dates[-numDaysToPlot:], np.clip(avgPctChannel[-numDaysToPlot:]*100.,-200.,200.), 'r-', lw=.1)
plt.plot( _dates[-numDaysToPlot:], numAboveBelowChannel[-numDaysToPlot:], 'b-', lw=.25)
plt.title("pyTAAA History Plot\nChannel Offset Signal")
plt.savefig(figure4path)
figure4path = 'PyTAAA_DailyChannelOffsetSignalV2.png' # re-set to name without full path
figure4_htmlText = "\n<br><h3>Channel Offset Signal</h3>\n"
figure4_htmlText = figure4_htmlText + "\nPlot shows up/down trending in last few days compared to trend for stocks in Nasdaq 100.\n"
figure4_htmlText = figure4_htmlText + '''<br><img src="'''+figure4path+'''" alt="PyTAAA by DonaldPG" width="850" height="500"><br>\n'''
###
### make a combined plot
### 2. make plot showing trend below B&H and trade-system Value
###
file3path = os.path.join( os.getcwd(), "pyTAAAweb_backtestPortfolioValue.params" )
backtestDate = []
backtestBHvalue = []
backtestSystemvalue = []
try:
with open( file3path, "r" ) as f:
# get number of lines in file
lines = f.read().split("\n")
numlines = len (lines)
for i in range(numlines):
try:
statusline = lines[i]
statusline_list = statusline.split(" ")
if len( statusline_list ) == 5:
backtestDate.append( datetime.datetime.strptime( statusline_list[0], '%Y-%m-%d') )
backtestBHvalue.append( float(statusline_list[2]) )
backtestSystemvalue.append( float(statusline_list[4]) )
except:
break
except:
print " Error: unable to read updates from pyTAAAweb_backtestPortfolioValue.params"
print ""
figure5path = os.path.join( os.getcwd(), "pyTAAA_web", "PyTAAA_backtestWithOffsetChannelSignal.png" )
plt.figure(5,figsize=(9,7))
plt.clf()
subplotsize = gridspec.GridSpec(2,1,height_ratios=[5,3])
plt.subplot(subplotsize[0])
plt.grid(True)
plt.yscale('log')
plotmax = 1.e10
plt.ylim([1000,max(10000,plotmax)])
numDaysToPlot = 252*10
numDaysToPlot = len( backtestBHvalue )
plt.plot( backtestDate[-numDaysToPlot:], backtestBHvalue[-numDaysToPlot:], 'r-', lw=1.25, label='Buy & Hold')
plt.plot( backtestDate[-numDaysToPlot:], backtestSystemvalue[-numDaysToPlot:], 'k-', lw=1.25, label='Trading System')
plt.legend(loc=2,prop={'size':9})
plt.title("pyTAAA History Plot\n Portfolio Value")
plt.text( backtestDate[-numDaysToPlot+50], 2500, "Backtest updated "+datetime.datetime.now().strftime("%A, %d. %B %Y %I:%M%p"), fontsize=7.5 )
plt.subplot(subplotsize[1])
plt.grid(True)
plt.ylim(-100, 100)
plt.plot( _dates[-numDaysToPlot:], np.clip(avgPctChannel[-numDaysToPlot:]*100.,-200.,200.), 'r-', lw=.1, label='avg Pct offset channel')
plt.plot( _dates[-numDaysToPlot:], numAboveBelowChannel[-numDaysToPlot:], 'b-', lw=.25, label='number above/below offset channel')
plt.legend(loc=3,prop={'size':6})
plt.savefig(figure5path)
figure5path = 'PyTAAA_backtestWithOffsetChannelSignal.png' # re-set to name without full path
figure5_htmlText = "\n<br><h3>Daily backtest with offset Channel trend signal</h3>\n"
figure5_htmlText = figure5_htmlText + "\nCombined backtest with offset Channel trend signal.\n"
figure5_htmlText = figure5_htmlText + '''<br><img src="'''+figure5path+'''" alt="PyTAAA by DonaldPG" width="850" height="500"><br>\n'''
return figure4_htmlText, figure5_htmlText
import countWords
import matplotlib.pylab as pylab
f = open('pg1661.txt', 'r')
s = f.read()
wordFreq = countWords.countFreqs(s)
freqs = wordFreq.values()
freqs = sorted(freqs)
freqs.reverse()
wordNumber = len(freqs)
pylab.xscale('log')
pylab.yscale('log')
pylab.plot(range(1, wordNumber + 1), freqs)
pylab.show()
def plot_BER():
BERarr1 = []
BERarr2 = []
BERarr3 = []
var1 = np.arange(3, 30, 0.5)
var2 = np.arange(30, 200, 10)
var3 = np.arange(200, 2200, 50)
var = np.hstack((var1, var2, var3)) # (0.1, 10, 0.05)
varx = np.log10(100 / var)
file = open('testdata.txt', 'w')
for i in var:
# print("正在产生高斯白噪声。。。")
AWGN_syb = generate_AWGN(i, 9997)
# print("成功生成高斯白噪声！")
# print("-----------------------------")
# print("正在产生信号。。。")
SIGS1_syb, SIGS2_syb, SIGtmp, SIGS_syb = generate_signal(9997)
BPSKS_syb, BPSKtmp, BPSK_syb = generate_BPSK(9997)
AWGN2_syb = []
for j in AWGN_syb:
AWGN2_syb.extend(j)
# print("成功生成信号！")
# print("-----------------------------")
# print("信号传输中。。。")
SIGR1_syb = add_AWGN(SIGS1_syb, AWGN_syb)
SIGR2_syb = add_AWGN(SIGS2_syb, AWGN_syb)
BPSKR_syb = add_AWGN(BPSKS_syb, AWGN2_syb)
# print("信号传输成功！")
# print("-----------------------------")
# print("接收端正在处理。。。")
SIGJ1_syb = get_SIGJ(SIGS_syb, SIGR1_syb, sybj1, SIGtmp)
SIGJ2_syb = get_SIGJ(SIGS_syb, SIGR2_syb, sybj2, SIGtmp)
BPSKJ_syb = get_BPSKJ(BPSK_syb, BPSKR_syb, bpskj, BPSKtmp)
# print("接收端处理完毕！")
# print("-----------------------------")
BER1 = get_BER(SIGS_syb, SIGJ1_syb)
BER2 = get_BER(SIGS_syb, SIGJ2_syb)
BERBPSK = get_BPSK_BER(BPSK_syb, BPSKJ_syb)
txt = str("归一化的信噪比（DB）：") + str(np.log10(
100 / i)) + str("DB") + "\n" + str("误比特率：") + "\n" + str(
"signal Ⅰ：") + str(BER1) + str(" signal Ⅱ(经过信道编码)：") + str(
BER2) + str(" BPSK：") + str(BERBPSK) + "\n" * 2
file.write(txt)
print("---------------------------------")
print("signal Ⅰ：")
print("信噪比：", np.log10(100 / i), " 误比特率：", BER1)
print("signal Ⅱ(经过信道编码)：")
print("信噪比：", np.log10(100 / i), " 误比特率：", BER2)
print("---------------------------------")
print("BPSK：")
print("信噪比：", np.log10(100 / i), " 误比特率：", BERBPSK)
print("---------------------------------")
BERarr1.append(BER1)
BERarr2.append(BER2)
BERarr3.append(BERBPSK)
# print ("发送端信号集一（符号）:\n", SIGS1_syb)
# print("噪声（符号）:\n", AWGN_syb)
# print ("接收端信号集一（符号）:\n", SIGR1_syb)
# print ("信号集一判决结果（符号）:\n", SIGJ1_syb)
#
# print ("发送端信号集二（符号）:\n", SIGS2_syb)
# print("噪声（符号）:\n", AWGN_syb)
# print ("接收端信号集二（符号）:\n", SIGR2_syb)
# print ("信号集二判决结果（符号）:\n", SIGJ2_syb)
# print("信号一的误比特率为： ", BER1)
# print("信号二的误比特率为： ", BER2)
file.close()
plot1 = plb.plot(varx, BERarr1, linestyle='-', color='b', label=u'SET 1')
plot2 = plb.plot(varx, BERarr2, linestyle='-', color='r', label=u'SET 2')
plot3 = plb.plot(varx, BERarr3, linestyle='-', color='g', label=u'BPSK')
plb.title(u'不同信噪比下的误码率') # give plot a title
plb.xlabel("Eb/N0")
plb.ylabel("BER")
plb.yscale('log')
plb.legend()
plb.show()
points_o1var = [[1e-5, 1e5], [1e-4, 1e5], [1e-4, 1e6]]
# points_o1var = [[1e-5, 1e-2], [2e-5, 2e-2], [1e-5, 2e-2]]
triangle_o1var = plt.Polygon(points_o1var, fill=None ,edgecolor='grey')
#VariancePlot
for eps_i in range (0,len(eps_list)):
plot_var = plt.plot(Nlist_inv, (sde_Jv_sq[eps_i] -sq_E_Jv[eps_i])/bins, lines[eps_i],
label =r'$\varepsilon=10^{-%d}$' %eps_list_exponents[eps_i] , linewidth=3)
plot_weighted = plt.plot(Nlist_inv, (sde_Jv_sq_weighted[eps_i] -sq_E_Jv_weighted[eps_i])/bins, lines[eps_i], linewidth=3.5)
plt.ylabel('Var ($\mathbf{\hat{Jv}} $)', fontsize = 16)
plt.xlabel('$1/N$', fontsize = 16)
plt.xscale(log_flag)
plt.yscale(log_flag)
# plt.legend([plot_var], loc='best')
#plt.legend(bbox_to_anchor=(0.95, 1), numpoints = 1 )
plt.gca().set_ylim([1e-4, 1e15])
plt.gca().add_patch(triangle_o1var)
plt.legend([plot_var], loc='best')
plt.legend(bbox_to_anchor=(1, 0.65), numpoints = 1 )
order= r'$\mathcal{O}(1/N)$'
plt.annotate(order, xy=(4.8e-5, 1.5e5), xytext=(4.8e-5, 1.5e5), fontsize=9, color='grey')
plt.annotate('with variance reduction', xy=(4e-6, 1e-1), xytext=(4e-6, 1e-1), fontsize=12,rotation=5.5, color='grey')
if(save_flag): plt.savefig("plots/Var_N_eps_nw.pdf")
plt.show()
# for eps_i in range (0,len(eps_list)):
# plot_var = plt.plot(Nlist_inv, (rho_sq -sq_E_rho)/bins, lines[eps_i],
# label =r'$\varepsilon=10^{-%d}$' %eps_list_exponents[eps_i] , linewidth=2)
#p.text(dp,co,"(%s%i,%s%i)"%(seq[res1-1],res1,seq[res2-1],res2),color=col)
rows.sort(key=lambda x: x[5], reverse=True)
for r in rows:
datafile.write(",".join([str(i) for i in r]) + "\n")
datafile.close()
#ymin, ymax = p.ylim()
p.annotate("A", (10, 10),
xytext=(0.1, 0.9),
textcoords='axes fraction',
color='blue')
p.annotate("B", (10, 10),
xytext=(0.9, 0.9),
textcoords='axes fraction',
color='red')
p.ylim(chainsep - 1, nres1 + 10)
p.grid()
#p.legend(loc=0)
p.xlabel("P(B) - P(A)")
p.ylabel("contact order")
p.title("A: %s\nB: %s" % (inpdb1, inpdb2))
p.yscale('log')
p.yticks([4, 5, 6, 7, 8, 9, 10, 20, 30, 40, 50, 60],
[4, 5, 6, 7, 8, 9, 10, 20, 30, 40, 50, 60])
outfname = mergeStrings(inpdb1, inpdb2)
p.savefig(outfname.replace("/", "_") + "_contact_diff.png", dpi=200)
#p.show()
def compute_prior_prob_smoothed(prior_prob_path,
prior_prob_smoothed_path,
sigma=5,
do_plot=True,
verbose=1):
"""
Interpolation on prior prob, next using interpolation to smoothness path, and normalize again
Reference: https://github.com/foamliu/Colorful-Image-Colorization/blob/master/class_rebal.py
Usage:
info = dict(
prior_prob_path = os.path.join(module_dir, "data", "prior_prob_train_div2k.npy"),
prior_prob_smoothed_path = os.path.join(module_dir, "data", "prior_prob_smoothed_train_div2k.npy"),
sigma = 5,
do_plot = True,
verbose = True,
)
locals().update(**info)
prior_prob_smoothed = compute_prior_prob_smoothed(**info)
"""
# load prior probability
if verbose == 1: print("\n=== Compute Prior Probability Smoothed === ")
prior_prob = np.load(prior_prob_path)
# add an epsilon to prior prob to avoid 0 vakues and possible NaN
prior_prob += 1E-3 * np.min(prior_prob)
# renormalize
prior_prob = prior_prob / (1.0 * np.sum(prior_prob))
# Smooth with gaussian
f = interp1d(np.arange(prior_prob.shape[0]), prior_prob)
xx = np.linspace(0, prior_prob.shape[0] - 1, 1000)
yy = f(xx)
window = gaussian(2000, sigma) # 2000 pts in the window, sigma=5
smoothed = convolve(yy, window / window.sum(), mode='same')
fout = interp1d(xx, smoothed)
prior_prob_smoothed = np.array(
[fout(i) for i in range(prior_prob.shape[0])])
prior_prob_smoothed = prior_prob_smoothed / np.sum(prior_prob_smoothed)
# Save
if prior_prob_smoothed_path is not None:
save_dir = os.path.dirname(prior_prob_smoothed_path)
if save_dir != "" and os.path.exists(save_dir) == False:
os.makedirs(save_dir)
np.save(prior_prob_smoothed_path, prior_prob_smoothed)
# if
if do_plot:
plt.figure(figsize=(20, 10))
plt.subplot(2, 2, 1)
plt.plot(prior_prob, label="prior_prob")
plt.plot(prior_prob_smoothed, "g--", label="prior_prob_smoothed")
plt.yscale("log")
plt.legend()
plt.subplot(2, 2, 2)
plt.plot(prior_prob, label="prior_prob")
plt.plot(xx, smoothed, "r-", label="smoothed")
plt.yscale("log")
plt.legend()
plt.subplot(2, 2, 3)
plt.hist(prior_prob, bins=100)
plt.xlabel("Prior probability")
plt.ylabel("Frequency")
plt.yscale("log")
plt.subplot(2, 2, 4)
plt.hist(prior_prob_smoothed, bins=100)
plt.xlabel("Prior probability smoothed")
plt.ylabel("Frequency")
plt.yscale("log")
plt.show()
# if
return prior_prob_smoothed
from sklearn.model_selection import train_test_split
from sklearn.svm import SVC
import matplotlib.pylab as plt
cancer = load_breast_cancer()
X_train, X_test, y_train, y_test = train_test_split(
cancer.data, cancer.target, random_state=0)
svc = SVC()
svc.fit(X_train, y_train)
print("Train Accuracy: {:.2f}".format(svc.score(X_train, y_train)))
print("Test Accuracy: {:.2f}".format(svc.score(X_test, y_test)))
plt.boxplot(X_train, manage_xticks=False)
plt.yscale("symlog")
plt.xlabel("feature list")
plt.ylabel("feature size")
plt.show()
###########################################################
min_on_training = X_train.min(axis=0)
range_on_training = (X_train - min_on_training).max(axis=0)
# 훈련 데이터에 최솟값을 빼고 범위로 나누면 각 특성에 대해 최솟값은 0 최댓값은 1 임
X_train_scaled = (X_train - min_on_training) / range_on_training
X_test_scaled = (X_test - min_on_training) / range_on_training
print("min:\n{}".format(X_train_scaled.min(axis=0)))
print("max:\n {}".format(X_train_scaled.max(axis=0)))
svc = SVC()
rankList = []
i = 1
for line in f:
s = line
word = s.split() # separating words and frequencies
wordList.append(word[0]) # storing words
freqList.append(word[1]) # storing frequencies
rankList.append(i) # storing ranks
i = i + 1
for i in range(0, len(freqList)):
freqList[i] = int(freqList[i])
rankList = np.log(rankList) # converting to log values
freqList = np.log(freqList)
pl.yscale("log") # defining log-log scale of graph
pl.xscale("log")
pl.plot(rankList, freqList)
# pl.axis([0,75000,164438,23135851162])
po = np.polyfit(rankList, freqList, 5) # for the smoothed graph
yfit = np.polyval(po, rankList)
pl.plot(rankList, yfit) # plotting the graph
pl.xlabel("Log Rank")
pl.ylabel("Log Frequency") # defining labels, titles and legends
pl.suptitle("Solution 3 : Frequency vs Rank")
pl.legend(["Log Frequency vs Log Rank", "Smoothened Graph"])
pl.show()
def compute_prior_factor(prior_prob_path,
prior_prob_smoothed_path,
prior_prob_factor_path,
gamma=0.5,
alpha=1,
do_plot=True,
verbose=1):
"""
Calculate prior probability factorization
Reference: https://github.com/foamliu/Colorful-Image-Colorization/blob/master/class_rebal.py
Usage:
info = dict(
prior_prob_path = os.path.join(data_dir, "preprocessing", "DIV2K", "prior_prob_train_div2k.npy"),
prior_prob_smoothed_path = os.path.join(data_dir, "preprocessing", "DIV2K", "prior_prob_smoothed_train_div2k.npy"),
prior_prob_factor_path = os.path.join(data_dir, "preprocessing", "DIV2K", "prior_prob_factor_train_div2k.npy"),
gamma = 0.5,
alpha = 1,
do_plot = True,
verbose = 1,
)
locals().update(**info)
prior_factor = compute_prior_factor(**info)
"""
if verbose == 1: print("\n=== Compute Prior Factor === ")
prior_prob = np.load(prior_prob_path)
prior_prob_smoothed = np.load(prior_prob_smoothed_path)
u = np.ones_like(prior_prob_smoothed)
u = u / np.sum(1.0 * u)
prior_factor = (1 - gamma) * prior_prob_smoothed + gamma * u
prior_factor = np.power(prior_factor, -alpha)
# renormalize
prior_factor = prior_factor / (np.sum(prior_factor * prior_prob_smoothed))
# Save
if prior_prob_factor_path is not None:
save_dir = os.path.dirname(prior_prob_factor_path)
if save_dir != "" and os.path.exists(save_dir) == False:
os.makedirs(save_dir)
np.save(prior_prob_factor_path, prior_factor)
# if
if do_plot:
plt.figure(figsize=(20, 10))
plt.subplot(1, 3, 1)
plt.hist(prior_prob)
plt.xlabel("Prior probability")
plt.ylabel("Frequency")
plt.yscale("log")
plt.subplot(1, 3, 2)
plt.hist(prior_prob_smoothed)
plt.xlabel("Prior probability smoothed")
plt.ylabel("Frequency")
plt.yscale("log")
plt.subplot(1, 3, 3)
plt.hist(prior_factor)
plt.xlabel("Prior probability smoothed factor")
plt.ylabel("Frequency")
plt.yscale("log")
plt.show()
# if
return prior_factor
E = LTB_E(r)
DL4 = (1.+z)**8*R**4*np.sin(theta)**2/np.sin(gamma)**2 * (
Rdash**2/R**2/(1.+2.*E)*dr_dgamma**2 + dtheta_dgamma**2)
DL = DL4**0.25
return DL4**0.25 - comp_dist
from matplotlib import pylab as plt
z = np.concatenate((np.logspace(np.log10(1e-1),np.log10(10),200),np.linspace(10.01,1100,800)))
fig = plt.figure()
for gamma in np.linspace(0.1,2*np.pi,7,endpoint=False):
plt.plot(z, lum_and_Ang_dist(gamma,z)[0],label=str(gamma))
plt.legend(loc='best')
plt.yscale('symlog',linthreshx=200)
#plt.xscale('log')
plt.xlim(z[0],z[-1])
fig = plt.figure()
for gamma in np.linspace(0.1,2*np.pi,7,endpoint=False):
plt.plot(z, lum_and_Ang_dist(gamma,z)[0],label=str(gamma))
plt.legend(loc='best')
plt.yscale('symlog',linthreshx=200.)
plt.xlim(z[0],z[-1])
#angular diameter distances
fig = plt.figure()
for gamma in np.linspace(0.1,2*np.pi,7,endpoint=False):
plt.plot(z, lum_and_Ang_dist(gamma,z)[1],label=str(gamma))
plt.legend(loc='best')
plt.xscale('log')
plt.xlim(z[0],z[-1])
def compute_prior_prob(image_files, width, height, do_plot, pts_in_hull_path,
prior_prob_path):
"""
Compute color prior probabilities for pts in hull
Reference: https://github.com/foamliu/Colorful-Image-Colorization/blob/master/class_rebal.py
Usage:
df_data = pd.read_hdf(os.path.join(data_dir, "preprocessing", "DIV2K", "div2k.hdf5"), "data")
list_types = ["'train'"]
df_select_data = df_data.query("type in [" + ",".join(list_types) + "]")
image_dir = os.path.join(dataset_dir, "DIV2K").replace("\\", "/")
image_files = image_dir + "/" + df_select_data["path"].values
image_files[0:3], len(image_files)
info = dict(
image_files = image_files,
pts_in_hull_path = os.path.join(data_dir, "colorization_richard_zhang", "pts_in_hull.npy"),
prior_prob_path = os.path.join(data_dir, "preprocessing", "DIV2K", "prior_prob_train_div2k.npy"),
width = 112,
height = 112,
do_plot = True
)
locals().update(**info)
prior_prob = compute_prior_prob(**info)
"""
# Load ab image
X_ab = []
for image_path in tqdm.tqdm(image_files):
result = read_image(image_path,
is_resize=True,
width=width,
height=height)
X_ab.append(result["res_img_Lab"][:, :, 1:])
# for
X_ab = np.array(X_ab)
X_ab = X_ab - 128.0
# Load the gamut points location
q_ab = np.load(pts_in_hull_path)
if do_plot:
plt.figure(figsize=(8, 8))
plt.title("ab quantize")
gs = gridspec.GridSpec(1, 1)
ax = plt.subplot(gs[0])
for i in range(q_ab.shape[0]):
ax.scatter(q_ab[:, 0], q_ab[:, 1])
ax.annotate(str(i), (q_ab[i, 0], q_ab[i, 1]), fontsize=6)
ax.set_xlim([-110, 110])
ax.set_ylim([-110, 110])
# for
# if
npts, c, h, w = X_ab.shape
X_a_ravel = np.ravel(X_ab[:, :, :, 0])
X_b_ravel = np.ravel(X_ab[:, :, :, 1])
X_ab_ravel = np.vstack((X_a_ravel, X_b_ravel)).T
if do_plot:
plt.title("Prior Distribution in ab space\n", fontsize=16)
plt.hist2d(X_ab_ravel[:, 0],
X_ab_ravel[:, 1],
bins=100,
density=True,
norm=LogNorm(),
cmap=plt.cm.jet)
plt.xlim([-120, 120])
plt.ylim([-120, 120])
plt.xticks(fontsize=12)
plt.yticks(fontsize=12)
plt.xlabel("b channel", fontsize=14)
plt.ylabel("a channel", fontsize=14)
plt.colorbar()
plt.show()
plt.clf()
plt.close()
# if
# Create nearest neighbord instance with index = q_ab
NN = 1
nearest = nn.NearestNeighbors(n_neighbors=NN,
algorithm='ball_tree').fit(q_ab)
# Find index of nearest neighbor for X_ab
dists, ind = nearest.kneighbors(X_ab_ravel)
# We now count the number of occurrences of each color
ind = np.ravel(ind)
counts = np.bincount(ind)
idxs = np.nonzero(counts)[0]
prior_prob = np.zeros((q_ab.shape[0]))
prior_prob[idxs] = counts[idxs]
# We turn this into a color probability
prior_prob = prior_prob / (1.0 * np.sum(prior_prob))
# Save
if prior_prob_path is not None:
save_dir = os.path.dirname(prior_prob_path)
if save_dir != "" and os.path.exists(save_dir) == False:
os.makedirs(save_dir)
pts_in_hull_name = os.path.basename(pts_in_hull_path)
safe_copy(pts_in_hull_path, os.path.join(save_dir, pts_in_hull_name))
np.save(prior_prob_path, prior_prob)
# if
if do_plot:
plt.hist(prior_prob, bins=100)
plt.xlabel("Prior probability")
plt.ylabel("Frequency")
plt.yscale("log")
plt.show()
# if
return prior_prob
pass
print "SQ = ",s
print "EQ = ",eq
print "SB1 = ",s1
print "EB1 = ",e1
print "SB2 = ",s2
print "EB2 = ",e2
ft=open(ftxt,'r')
l=ft.readlines()
ft.close()
eList=[]
sList=[]
for i in range(0,len(l)):
eList.append(float(l[i].split()[0]))
sList.append((l[i].split()[1]))
data_x = [eq,e1,e2]
data_y = [sq,s1,s2]
P.figure(1)
P.yscale('log')
P.plot(eList,sList)
P.plot(data_x,data_y,'or')
P.savefig(pre +".png")
def hierarchical_clustering_ShiMalik(self, K, plots=False):
"""
Implementation of hierarchical clustering according to
Shi & Malik 2000.
At each iteration, one cluster is added, minimizing the global NCut.
We implement this by computing the increase in the coherence ratio
rho and choose the maximum increase. This is
equivalent to minimizing the global NCut, cf. eq. A2
"""
networks = {}
networks[0] = [self]
boolean = True
i = 0
while (i < K and boolean):
i += 1
print('Level: ', i)
optimal_drhos = []
optimal_cutoffs = []
for j in range(len(networks[i - 1])):
nw = networks[i - 1][j]
if nw.N < 100:
optimal_drhos.append(np.nan)
optimal_cutoffs.append(np.nan)
continue
nw.compute_laplacian_spectrum()
V_fiedler = nw.Lsym_eigenvectors[:, 1]
c_range = np.linspace(np.min(V_fiedler), np.max(V_fiedler),
100)[1:]
drhos = []
for c in c_range:
indices_1 = np.argwhere(V_fiedler <= c)[:, 0]
indices_2 = np.argwhere(V_fiedler > c)[:, 0]
drhos.append(nw.drho_split(indices_1, indices_2))
drhos = np.array(drhos)
if plots:
plt.plot(c_range, drhos)
plt.yscale('log')
plt.grid(True)
plt.title(r'$\Delta \rho_{global}$ \
for different cutoffs. Network' + str(i) +
str(j))
plt.show()
cutoff_opt = c_range[np.nanargmax(drhos)]
print('Choosing as cutoff: ', str(cutoff_opt))
optimal_drhos.append(np.nanmax(drhos))
optimal_cutoffs.append(cutoff_opt)
if (np.isnan(optimal_drhos).all()):
boolean = False
else:
i_cluster = np.nanargmax(optimal_drhos)
print('Splitting cluster ', i_cluster + 1)
cutoff_cluster = optimal_cutoffs[np.nanargmax(optimal_drhos)]
nw_to_split = networks[i - 1][i_cluster]
V_fiedler = nw_to_split.Lsym_eigenvectors[:, 1]
indices_1 = np.argwhere(V_fiedler <= cutoff_cluster)[:, 0]
indices_2 = np.argwhere(V_fiedler > cutoff_cluster)[:, 0]
#If a cluster is split, the largest sub-cluster receives the same label.
if len(indices_1) < len(indices_2):
ind_ = indices_1.copy()
indices_1 = indices_2.copy()
indices_2 = ind_
adjacency_matrix_1 = nw_to_split.adjacency_matrix[
indices_1, :][:, indices_1]
adjacency_matrix_2 = nw_to_split.adjacency_matrix[
indices_2, :][:, indices_2]
cluster_indices_1 = nw_to_split.cluster_indices[indices_1]
cluster_indices_2 = nw_to_split.cluster_indices[indices_2]
cluster_volume_1 = nw_to_split.cluster_volume[indices_1]
cluster_volume_2 = nw_to_split.cluster_volume[indices_2]
cluster_label_1 = nw_to_split.cluster_label
old_labels = [nw.cluster_label for nw in networks[i - 1]]
cluster_label_2 = np.max(old_labels) + 1
network_children = [
undirected_network(adjacency_matrix_1, cluster_indices_1,
cluster_volume_1, cluster_label_1),
undirected_network(adjacency_matrix_2, cluster_indices_2,
cluster_volume_2, cluster_label_2)
]
networks[i] = networks[i - 1].copy()
networks[i].pop(i_cluster)
networks[i] += network_children #append in the end
self.clustered_networks = networks
tsmap_P8_P301_Error95 = numpy.array(tsmap_P8_P301_Error95).astype(float)
# Set the plotting format
try:
IDL.plotformat()
# Plot P7_P203 TS vs P8_P301 TS
GRB130427A = numpy.where(GRBs == '130427324')
good = numpy.where((tsmap_P7_P203_MaxTS > 0) & (tsmap_P8_P301_MaxTS > 0))[0]
plt.scatter(tsmap_P7_P203_MaxTS[good], tsmap_P8_P301_MaxTS[good])
plt.annotate('GRB130427A', xy=(tsmap_P7_P203_MaxTS[GRB130427A],tsmap_P8_P301_MaxTS[GRB130427A]), xytext=(-35,10), textcoords='offset points', ha='center', va='bottom')
plt.plot([1,10000],[1,10000], '--')
plt.xscale('log')
plt.yscale('log')
plt.xlim(1,10000)
plt.ylim(1,10000)
plt.xlabel('TS (P7_203)')
plt.ylabel('TS (P8_301)')
plt.show()
# Plot P7_P203 90% Error vs P8_P301 90% Error
good = numpy.where((tsmap_P7_P203_MaxTS > 0) & (tsmap_P8_P301_MaxTS > 0))[0]
plt.scatter(tsmap_P7_P203_Error90[good], tsmap_P8_P301_Error90[good], c=numpy.log10(tsmap_P8_P301_MaxTS[good]))
plt.annotate('GRB130427A', xy=(tsmap_P7_P203_Error90[GRB130427A],tsmap_P8_P301_Error90[GRB130427A]), xytext=(40,-10), textcoords='offset points', ha='center', va='bottom')
cbar = plt.colorbar(pad = 0.02)
cbar.set_label(r'log TS$_{\rm P8}$')
plt.plot([0.001,10],[0.001,10], '--')
plt.xlim(0.01,10)
plt.ylim(0.01,10)
def makeTrendDispersionPlot( ):
from functions.TAfunctions import SMA, MoveMax
import functions.allstats
##########################################
# read uptrending stocks status file and make plot
##########################################
file4path = os.path.join( os.getcwd(), "pyTAAAweb_MeanTrendDispersion_status.params" )
figure4path = os.path.join( os.getcwd(), "pyTAAA_web", "PyTAAA_MeanTrendDispersion.png" )
dateMedians = []
valueMeans = []
valueMedians = []
try:
with open( file4path, "r" ) as f:
# get number of lines in file
lines = f.read().split("\n")
numlines = len (lines)
for i in range(numlines):
try:
statusline = lines[i]
statusline_list = statusline.split(" ")
if len( statusline_list ) == 5:
dateMedians.append( datetime.datetime.strptime( statusline_list[0], '%Y-%m-%d') )
valueMeans.append( float(statusline_list[2]) )
valueMedians.append( float(statusline_list[4]) )
except:
break
except:
print " Error: unable to read updates from pyTAAAweb_MeanTrendDispersion_status.params"
print ""
valueMeans = np.array( valueMeans ).astype('float')
valueMeansSMA = SMA( valueMeans, 100 )
valueMedians = np.array( np.clip(valueMedians,-25.,25. ) ).astype('float')
valueMediansSMA = SMA( valueMedians, 100 )
plt.figure(4,figsize=(9,7))
plt.clf()
plt.grid(True)
numDaysToPlot = 252*10
numDaysToPlot = len( valueMeans )
plt.plot( dateMedians[-numDaysToPlot:], valueMediansSMA[-numDaysToPlot:], 'r-', lw=1)
plt.plot( dateMedians[-numDaysToPlot:], valueMedians[-numDaysToPlot:], 'r-', lw=.25)
plt.title("pyTAAA History Plot\n Average Trend Dispersion")
plt.savefig(figure4path)
figure4path = 'PyTAAA_MeanTrendDispersion.png' # re-set to name without full path
figure4_htmlText = "\n<br><h3>Average Trend Dispersion</h3>\n"
figure4_htmlText = figure4_htmlText + "\nPlot shows dispersion of trend for stocks in Nasdaq 100 (thin line), 100 day moving average (thick line).\n"
figure4_htmlText = figure4_htmlText + "\nBlack lines use means, red lines use average sharpe.\n"
figure4_htmlText = figure4_htmlText + '''<br><img src="'''+figure4path+'''" alt="PyTAAA by DonaldPG" width="850" height="500"><br>\n'''
###
### make a combined plot
### 1. get percent of uptrending stocks
###
file2path = os.path.join( os.getcwd(), "pyTAAAweb_numberUptrendingStocks_status.params" )
date = []
value = []
active = []
try:
with open( file2path, "r" ) as f:
# get number of lines in file
lines = f.read().split("\n")
numlines = len (lines)
for i in range(numlines):
try:
statusline = lines[i]
statusline_list = statusline.split(" ")
if len( statusline_list ) == 5:
date.append( datetime.datetime.strptime( statusline_list[0], '%Y-%m-%d') )
value.append( float(statusline_list[2]) )
active.append( float(statusline_list[4]) )
except:
break
except:
print " Error: unable to read updates from pyTAAAweb_numberUptrendingStocks_status.params"
print ""
value = np.array( value ).astype('float') / np.array( active ).astype('float')
valueSMA = SMA( value, 100 )
valueMMA = MoveMax( valueSMA, 252 )
valueSMA2 = SMA( value, 500 )
valueMMA2 = MoveMax( valueSMA2, 252 )
PctInvested = 1.0+(value-valueSMA+value-valueSMA2)-(valueMMA2-valueSMA2 + valueMMA-valueSMA)
valueMediansSMA65 = SMA( valueMedians, 65 )
valueMediansSMA100 = SMA( valueMedians, 100 )
valueMediansSMA65 = ( valueMediansSMA65 - valueMediansSMA65.mean() ) * 8. + .7
valueMediansSMA100 = ( valueMediansSMA100 - valueMediansSMA100.mean() ) * 8. + .7
file3path = os.path.join( os.getcwd(), "pyTAAAweb_backtestPortfolioValue.params" )
backtestDate = []
backtestBHvalue = []
backtestSystemvalue = []
try:
with open( file3path, "r" ) as f:
# get number of lines in file
lines = f.read().split("\n")
numlines = len (lines)
for i in range(numlines):
try:
statusline = lines[i]
statusline_list = statusline.split(" ")
if len( statusline_list ) == 5:
backtestDate.append( datetime.datetime.strptime( statusline_list[0], '%Y-%m-%d') )
backtestBHvalue.append( float(statusline_list[2]) )
backtestSystemvalue.append( float(statusline_list[4]) )
except:
break
except:
print " Error: unable to read updates from pyTAAAweb_backtestPortfolioValue.params"
print ""
###
### make a combined plot
### 2. make plot showing trend below B&H and trade-system Value
###
figure5path = os.path.join( os.getcwd(), "pyTAAA_web", "PyTAAA_backtestWithTrend.png" )
plt.figure(5,figsize=(9,7))
plt.clf()
subplotsize = gridspec.GridSpec(2,1,height_ratios=[5,3])
plt.subplot(subplotsize[0])
plt.grid(True)
plt.yscale('log')
plotmax = 1.e10
plt.ylim([1000,max(10000,plotmax)])
numDaysToPlot = 252*10
numDaysToPlot = len( backtestBHvalue )
plt.plot( backtestDate[-numDaysToPlot:], backtestBHvalue[-numDaysToPlot:], 'r-', lw=1.25, label='Buy & Hold')
plt.plot( backtestDate[-numDaysToPlot:], backtestSystemvalue[-numDaysToPlot:], 'k-', lw=1.25, label='Trading System')
plt.legend(loc=2,prop={'size':9})
plt.title("pyTAAA History Plot\n Portfolio Value")
plt.text( backtestDate[-numDaysToPlot+50], 2500, "Backtest updated "+datetime.datetime.now().strftime("%A, %d. %B %Y %I:%M%p"), fontsize=7.5 )
plt.subplot(subplotsize[1])
plt.grid(True)
plt.ylim(0, 1.2)
numDaysToPlot = 252*10
numDaysToPlot = len( value )
plt.plot( date[-numDaysToPlot:], value[-numDaysToPlot:], 'k-', lw=.25, label='Percent Uptrending')
plt.plot( date[-numDaysToPlot:], np.clip(PctInvested[-numDaysToPlot:],0.,1.2), 'g-', alpha=.65, lw=.5, label='Percent to Invest')
numDaysToPlot = len( valueMedians )
plt.legend(loc=3,prop={'size':6})
plt.savefig(figure5path)
figure5path = 'PyTAAA_backtestWithTrend.png' # re-set to name without full path
figure5_htmlText = "\n<br><h3>Daily backtest with trend indicators</h3>\n"
figure5_htmlText = figure5_htmlText + "\nCombined backtest with Trend indicators.\n"
figure5_htmlText = figure5_htmlText + '''<br><img src="'''+figure5path+'''" alt="PyTAAA by DonaldPG" width="850" height="500"><br>\n'''
return figure5_htmlText
