def plot(self, debug = False):
"""plot figures for population, nuisance parameters"""
# first figure out what scheme is used
self.list_scheme()
# next get MABR sampling done
self.MBAR_analysis()
# load in precomputed P and dP from MBAR analysis
pops0, pops1 = self.P_dP[:,0], self.P_dP[:,self.K-1]
dpops0, dpops1 = self.P_dP[:,self.K], self.P_dP[:,2*self.K-1]
t0 = self.traj[0]
t1 = self.traj[self.K-1]
# Figure Plot SETTINGS
label_fontsize = 12
legend_fontsize = 10
fontfamily={'family':'sans-serif','sans-serif':['Arial']}
plt.rc('font', **fontfamily)
# determine number of row and column
if (len(self.scheme)+1)%2 != 0:
c,r = 2, (len(self.scheme)+2)/2
else:
c,r = 2, (len(self.scheme)+1)/2
plt.figure( figsize=(4*c,5*r) )
# Make a subplot in the upper left
plt.subplot(r,c,1)
plt.errorbar( pops0, pops1, xerr=dpops0, yerr=dpops1, fmt='k.')
plt.hold(True)
plt.plot([1e-6, 1], [1e-6, 1], color='k', linestyle='-', linewidth=2)
plt.xlim(1e-6, 1.)
plt.ylim(1e-6, 1.)
plt.xlabel('$p_i$ (exp)', fontsize=label_fontsize)
plt.ylabel('$p_i$ (sim+exp)', fontsize=label_fontsize)
plt.xscale('log')
plt.yscale('log')
# label key states
plt.hold(True)
for i in range(len(pops1)):
if (i==0) or (pops1[i] > 0.05):
plt.text( pops0[i], pops1[i], str(i), color='g' )
for k in range(len(self.scheme)):
plt.subplot(r,c,k+2)
plt.step(t0['allowed_'+self.scheme[k]], t0['sampled_'+self.scheme[k]], 'b-')
plt.hold(True)
plt.xlim(0,5)
plt.step(t1['allowed_'+self.scheme[k]], t1['sampled_'+self.scheme[k]], 'r-')
plt.legend(['exp', 'sim+exp'], fontsize=legend_fontsize)
if self.scheme[k].find('cs') == -1:
plt.xlabel("$\%s$"%self.scheme[k], fontsize=label_fontsize)
plt.ylabel("$P(\%s)$"%self.scheme[k], fontsize=label_fontsize)
plt.yticks([])
else:
plt.xlabel("$\sigma_{%s}$"%self.scheme[k][6:],fontsize=label_fontsize)
plt.ylabel("$P(\sigma_{%s})$"%self.scheme[k][6:],fontsize=label_fontsize)
plt.yticks([])
plt.tight_layout()
plt.savefig(self.picfile)
def ecdf_by_observed_label(y_true, y_pred):
"""Plots the empirical cumulative density functions by observed label.
Parameters
----------
y_true : array_like
Observed labels, either 0 or 1.
y_pred : array_like
Predicted probabilities, floats on [0, 1].
Notes
-----
.. plot:: pyplots/ecdf_by_observed_label.py
"""
x = np.linspace(0, 1)
ecdf = ECDF(y_pred[y_true == 0])
y_0 = ecdf(x)
ecdf = ECDF(y_pred[y_true == 1])
y_1 = ecdf(x)
plt.step(x, y_0, label='Observed label 0')
plt.step(x, y_1, label='Observed label 1')
plt.xlabel('Predicted Probability')
plt.ylabel('Proportion')
plt.title('Empirical Cumulative Density Functions by Observed Label')
plt.legend(loc='lower right')
def plotSegments(jason, name="", labels=False):
# read delimeters for each stop
delims = readDelims()
stopAdds = [getStopSet(jason["packets"], stop) for stop in delims]
intersectBins = [set() for stop in delims]
combobs = combinations(range(len(delims)), 2)
for combob in combobs:
curSect = stopAdds[combob[0]].intersection(stopAdds[combob[1]])
for i in range(combob[0], combob[1]):
intersectBins[i+1].update(curSect)
probably_junk = stopAdds[0].intersection(stopAdds[-1])
y = [len(bin - probably_junk) + 2 for bin in intersectBins]
y[0] = y[1] # For labelling purposes
x = [stop['start'] for stop in delims]
realy = [stop['actual'] for stop in delims]
plot.xlabel('Seconds since '+jason["initial_time"])
plot.ylabel('Number of bus occupants (predicted)')
plot.xlim(0, delims[-1]['end'])
plot.title(name)
plot.step(x, y)
plot.step(x, realy, color="purple", where="post")
if(labels):
for stop in delims:
annotate(stop["code"], stop["start"], stop["actual"], 10, 10)
makeWidePlot("bus", "segments")
plot.show()
def testExponaverage(self):
"""
generate and fit a histogram of an exponential distribution with many
events using the average time to find the fit. The histogram is then
saved in testExponaverage.png
"""
tau = 25.0
nBins = 400
size = 100
taulist = []
for i in range(1000):
x = range(nBins)
yPlot = funcExpon(xPlot, *popt)
timeHgValues = np.zeros(nBins, dtype=np.int64)
timeStamps = expon.rvs(loc=0, scale=tau, size=size)
ts64 = timeStamps.astype(np.uint64)
tsBinner.tsBinner(ts64, timeHgValues)
param = sum(timeStamps)/len(timeStamps)
fit = expon.pdf(x,param)
fit *= size
taulist.append(param)
hist,bins = np.histogram(taulist, bins=20, range=(15,35))
width = 0.7*(bins[1]-bins[0])
center = (bins[:-1]+bins[1:])/2
#plt.bar(center, hist, align = 'center', width = width) produces bar graph
plt.step(center, hist, where = 'post')
plt.savefig(inspect.stack()[0][3]+".png")
def mask_spectrum(flux_to_fit,interactive=True,mask_lower_limit=None,mask_upper_limit=None):
"""
Interactively and iteratively creates a Boolean mask for a spectrum.
"""
if interactive:
plt.ion()
continue_parameter = 'no'
mask_switch = 'yes'
while mask_switch == 'yes':
pixel_array = np.arange(len(flux_to_fit))
plt.figure()
plt.step(pixel_array,flux_to_fit)
mask_lower_limit_string = raw_input("Enter mask lower limit (in pixels): ")
mask_lower_limit = float(mask_lower_limit_string)
mask_upper_limit_string = raw_input("Enter mask upper limit (in pixels): ")
mask_upper_limit = float(mask_upper_limit_string)
mask = (pixel_array >= mask_lower_limit) & (pixel_array <= mask_upper_limit)
flux_to_fit_masked = np.ma.masked_where(mask,flux_to_fit)
plt.step(pixel_array,flux_to_fit_masked)
continue_parameter = raw_input("Happy? (yes or no]): ")
plt.close()
if continue_parameter == 'yes':
mask_switch = 'no'
else:
pixel_array = np.arange(len(flux_to_fit))
mask = (pixel_array >= mask_lower_limit) & (pixel_array <= mask_upper_limit)
return mask
def doit(dataset, sigma1, C1, sigma2, C2, weights_factor):
dyear = dataset.dyear
clear = dataset.data['Mask'][:,line,sample]
x = dataset.dyear[clear]
y = dataset.data[band][:,line,sample][clear]/100. # TOA-reflectance in %
# outlier detection #
# fit SVR and exclude all observations with residual >= res_max
# sigma = 3 month and C=15% TOA reflectance works well
res_max = 15.
fsvr = svr(x, y, sigma=sigma1, C=C1, epsilon=1.)
valid = abs(fsvr(x)-y) <= res_max
# fit Zhu
fzhu = zhu(x[valid], y[valid])
# fit SVR on residuals of Zhu; weight residuals according to the data density
density_kernel_size = 25.
xdensity = scipy.ndimage.filters.convolve1d(1*clear, ones(density_kernel_size), mode='constant', cval=0) #(25-1)*8/30. = 6.4 month window
# weights_factor = 2. # stretch range is [1..1+factor]
weights = 1+(density_kernel_size-xdensity)/density_kernel_size*weights_factor # >10/25->weigth=1; 0/25->weight=2*factor; the rest scales in between
yr = y[valid]-fzhu(x[valid])
fsvr2 = svr(x[valid], yr, sigma=sigma2, C=C2, epsilon=0.001, weights=weights[clear][valid])
# Zhu+SVR ensemble
fzhusvr = lambda x: fzhu(x)+fsvr2(x)
# plot results
# plt.figure(figsize=[10,4])
plt.plot(x, y, 'bo', alpha=0.3)
plt.step(dyear, xdensity, 'k-', label='data densitiy', alpha=0.3)
plt.plot(dyear, fzhu(dyear), 'g-', label='Zhu et al.', alpha=0.7)
plt.plot(dyear, fsvr(dyear), 'y-', label='SVR', alpha=0.7)
plt.plot(dyear, fzhusvr(dyear), 'r-', label='SVR+Zhu', alpha=0.7, linewidth=2)
plt.legend()
def testExponManyEvents(self):
"""
generate and fit an exponential distribution with lifetime of 25
make a plot in testExponManyEvents.png
"""
tau = 25.0
nBins = 400
size = 100
taulist = []
for i in range(1000):
x = range(nBins)
timeHgValues = np.zeros(nBins, dtype=np.int64)
timeStamps = expon.rvs(loc=0, scale=tau, size=size)
ts64 = timeStamps.astype(np.uint64)
tsBinner.tsBinner(ts64, timeHgValues)
param = expon.fit(timeStamps)
fit = expon.pdf(x,loc=param[0],scale=param[1])
fit *= size
print "i=",i," param[1]=",param[1]
taulist.append(param[1])
hist,bins = np.histogram(taulist, bins=20, range=(15,25))
width = 0.7*(bins[1]-bins[0])
center = (bins[:-1]+bins[1:])/2
plt.step(center, hist, where = 'post')
plt.savefig(inspect.stack()[0][3]+".png")
def matplotlibExample():
x = np.linspace(10, 50, 100)
y = np.cos(x) / x
plt.step(x, y, "co")
plt.show()
def draw_eta_profile(analyzed_root_files, # path to edm-analyzed root files
gen_energy, # energy with which particle is generated
mylabel = "", # label for histograms
histcolor = '#000000'):
######################################################################################
# Function reads in the root files with edm-analyzed data, #
# chains the data from the different files to one root tree, #
# reads the relevant data to arrays #
# and creates and draws histograms. #
# Drawing of errobars is done here. #
# Settings for decorations such as axeslabels are not done here, but in main program #
######################################################################################
t0 = time.time()
## add to tree all edm-analyzed reco files
rh_tree = ROOT.TChain("demo/rh_tree")
rh_tree.Add(analyzed_root_files)
print "=========================================================="
print "Events in",str(analyzed_root_files),": ", rh_tree.GetEntries()
## initialize arrays
particle_etas = [] # list with all eta values in cut intervall
eta_energies = [] # list with all corresponding total RecHit energies for each eta over all events
## loop over all gen_particles
for i in range(rh_tree.GetEntries()):
rh_tree.GetEntry(i)
momentum_vector = ROOT.TVector3(rh_tree.gen_part_momentum_x,
rh_tree.gen_part_momentum_y,
rh_tree.gen_part_momentum_z)
eta = momentum_vector.Eta()
## do eta cut
if ((eta > min_eta) and (eta < max_eta)):
# and ((rh_tree.ecal_total_energy+rh_tree.hcal_total_energy) < 40.)):
particle_etas.append(eta)
eta_energies.append(rh_tree.ecal_total_energy
+ rh_tree.hcal_total_energy)
particle_etas = np.array(particle_etas,dtype=float)
eta_energies = np.array(eta_energies,dtype=float)
cut_event_numb = particle_etas.size
print "time needed to read in data and loop over entries: ", np.round(time.time()-t0,2)
## draw histogram
# etas are histogrammed automatically, corresponding energies are used as weights
binvalues, binedges, binerrors = calc_histogram(cut_event_numb,
particle_etas,
eta_energies/float(gen_energy))
t1 = time.time()
plt.step(binedges, np.append(binvalues, binvalues[-1]),
where="post", color=histcolor,
label=str(mylabel))
plt.errorbar(np.array(binedges[:-1])+(binedges[1]-binedges[0])/2, binvalues,
yerr=binerrors,
fmt='none', ecolor=histcolor)
print "time needed to draw plot: ", np.round(time.time()-t1,2)
return binvalues, binerrors, binedges
def line_flux(self,EL_wave,wave_range=None,plot=False): ''' -----PURPOSE----- Extract the flux from the emission line from the 'SPEC1D_NORMIVAR' flux density -----INPUT------- EL_wave Cental wavelength of the emission line [Angstroms] wave_range [min,max] where min and max are the wavelenghts in Angstroms that start and end the range over which you want to fit. By default (None) will use [EL_wave-25, EL_wave+25] plot if True, will plot the flux density and the 1D Gaussian fit -----OUTPUT------ line_flux In erg/cm^2/s ''' if wave_range == None: wave_range = [EL_wave-25,EL_wave+25] waves_line, g = self.fit_gaussian(EL_wave=EL_wave,wave_range=wave_range) line_flux = sum(g(waves_line)) # print "Line flux is %.4g erg/s/cm^2" % line_flux wave,spec1D_calibrated = self.extract_normalized_spectrum(spectype='SPEC1D_NORMIVAR') line_mask = np.logical_and(wave >= min(wave_range),wave <= max(wave_range)) waves_line = wave[line_mask] spec1D_line = spec1D_calibrated[line_mask] if plot: plt.step(waves_line,spec1D_line,color='r',label='flux density') plt.plot(waves_line,g(waves_line),color='b',label='Gaussian fit') plt.legend() return line_flux
def main():
'''The data in this example give the life talbe for motion sickness data
from an experiment with vertical movement at a frequency of 0.167 Hz and
acceleration 0.111 g, and of a second experiment with 0.333 Hz and acceleration
of 0.222 g.
'''
# get the data
data1 = getData('altman_13_2.txt', subDir='..\Data\data_altman')
data2 = getData('altman_13_3.txt', subDir='..\Data\data_altman')
# Determine the Kaplan-Meier curves
(p1, r1, t1, sp1,se1) = kaplanmeier(data1)
(p2, r2, t2, sp2,se2) = kaplanmeier(data2)
# Make a combined plot for both datasets
plt.step(t1,sp1, where='post')
plt.hold(True)
plt.step(t2,sp2,'r', where='post')
plt.legend(['Data1', 'Data2'])
plt.ylim(0,1)
plt.xlabel('Time')
plt.ylabel('Survival Probability')
plt.show()
# Check the hypothesis that the two survival curves are the same
# --- >>> START stats <<< ---
(p, X2) = logrank(data1, data2)
# --- >>> STOP stats <<< ---
return p # supposed to be 0.073326322306832212
def plotCDF(list_elements, sim_run, repetition, actual_capacity, TOFILE):
N = len(list_elements)
x = numpy.sort(list_elements)
y = numpy.linspace(0, 1, N)
plt.step(x, y * 100)
plt.title('Simulation %d-%d with capacity of %d' %
(sim_run + 1, repetition + 1, actual_capacity))
plt.xlabel('average time in system [tu]')
# displaying tick range. Adjustments needed for long run simulations!
plt.ylim(0, 100)
if (max(x) >= 500):
tickrange = 50
elif (500 > max(x) >= 100):
tickrange = 25
elif (100 > max(x) >= 20):
tickrange = 5
else:
tickrange = 1
plt.xticks(range(0, int(max(x))+2, tickrange))
plt.yticks(range(0, 100, 10))
plt.ylim(ymax=101)
plt.grid()
if (TOFILE):
plt.tight_layout()
plt.savefig('./figs/cdf/time_in_sys/simulation%d-%d.pdf' %
(sim_run + 1, repetition + 1), dpi=(150), format='pdf')
else:
plt.show()
plt.close()
def values_fromCDF():
'''Calculate an empirical cumulative distribution function, compare it with the exact one, and
find the exact point for a specific data value.'''
# Generate normally distributed random data
myMean = 5
mySD = 2
numData = 100
data = stats.norm.rvs(myMean, mySD, size=numData)
# Calculate the cumulative distribution function, CDF
numbins = 20
counts, bin_edges = np.histogram(data, bins=numbins, normed=True)
cdf = np.cumsum(counts)
cdf /= np.max(cdf)
# compare with the exact CDF
plt.step(bin_edges[1:],cdf)
plt.hold(True)
plt.plot(x, stats.norm.cdf(x, myMean, mySD),'r')
# Find out the value corresponding to the x-th percentile: the
# "cumulative distribution function"
value = 2
myMean = 5
mySD = 2
cdf = stats.norm.cdf(value, myMean, mySD)
print(('With a threshold of {0:4.2f}, you get {1}% of the data'.format(value, round(cdf*100))))
# For the percentile corresponding to a certain value:
# the "inverse cumulative distribution function"
value = 0.025
icdf = stats.norm.isf(value, myMean, mySD)
print(('To get {0}% of the data, you need a threshold of {1:4.2f}.'.format((1-value)*100, icdf)))
plt.show()
def plot_allocation(self):
"""
This function ...
:return:
"""
# Determine the path to the plot file
plot_path = fs.join(self.output_path, "allocation.pdf")
# Initialize figure
plt.figure()
plt.clf()
# Plot the memory usage of the root process
plt.plot(self.data[0].times, self.data[0].memory, label="total memory usage")
# Plot the memory allocation of the root process
plt.step(self.allocation.times, self.allocation.cumulative, where="post", linestyle="--", label="allocated array memory")
# Set the axis labels
plt.xlabel("Time (s)", fontsize='large')
plt.ylabel("Memory usage (GB)", fontsize='large')
# Set the plot title
plt.title("Memory (de)allocation")
# Set the legend
plt.legend(loc='lower right', prop={'size': 8})
# Save the figure
plt.savefig(plot_path, bbox_inches='tight', pad_inches=0.25)
plt.close()
def plot_concurrency(trans_stats, filename):
sorted_keys = trans_stats.keys()
sorted_keys.sort()
# outfilename = filename[0:filename.rfind(".")]+'_concur.csv'
# outfile = open(outfilename, "w")
for key in sorted_keys:
max_time = 65000
trans_stats[key].sort()
step = []
concur = []
for i in xrange(0, 65000, 10):
step.append(i)
pos = 0
c = 0
while (pos < len(trans_stats[key])):
if (trans_stats[key][pos][0] > i):
break
c = trans_stats[key][pos][1]
pos += 1
concur.append(c)
# line = ("%d,%d") % (i,c)
# outfile.write(line)
plt.step(step, concur)
label='dest_'+key
# plt.xlim(0, max_time)
# plt.ylim(0, max_conr+1)
plt.legend()
outfile = filename[0:filename.rfind(".")]+'_concur.png'
plt.savefig(outfile)
def plot_log_distribution(edges, hist, fit, threshold, line_label, xlabel, title, old_threshold=0):
import matplotlib.pyplot as plt
plt.clf()
# stretch bars, so can run off below 0
plot_hist = np.array([np.log10(x) if x != 0 else -1 for x in hist])
plt.step(edges[1:], plot_hist, color="k", label=line_label)
plt.plot(edges, fit, "b-", label="best fit")
plt.xlabel(xlabel)
plt.ylabel("log10(Frequency)")
# set y-lim to something sensible
plt.ylim([-0.3, max(plot_hist)])
plt.xlim([0, max(edges)])
plt.axvline(threshold, c="r", label="threshold = {}".format(threshold))
if old_threshold != 0:
plt.axvline(old_threshold, c="g", label="old threshold = {}".format(old_threshold))
plt.legend(loc="upper right")
plt.title(title)
plt.show()
return # plot_log_distribution
def plotControls(self,n=None,save=None):
t = NP.linspace(0.0,self.MPC.total_plant-self.MPC.t_plant,self.MPC.N_sampling)
if n is None:
np = NP.ceil(NP.sqrt(NP.abs(self.MPC.nu)))
for i in range(self.MPC.nu):
plt.subplot(np,np,i+1)
plt.step(t,self.u_final[i,:])
plt.xlabel('t')
plt.ylabel('u'+str(i))
if save is not None:
plt.savefig(save[0]+'controls',format=save[1])
plt.show()
elif isinstance(n,list) is True:
if len(n)>self.MPC.nu:
raise IOError('List of controls too long')
np = NP.ceil(NP.sqrt(NP.abs(len(n))))
j = 1
for i in n:
plt.subplot(np,np,j)
plt.plot(t,self.u_final[i,:])
plt.xlabel('t')
plt.ylabel('u'+str(i))
j+=1
if save is not None:
plt.savefig(save[0]+'_controls',format=save[1])
plt.show()
elif isinstance(n,int) is True:
plt.plot(t,self.u_final[n,:])
plt.xlabel('t')
plt.ylabel('u'+str(n))
if save is not None:
plt.savefig(save[0]+'_controls',format=save[1])
plt.show()
else:
raise IOError('\'n\' should be a list or integer')
def plot_hists_lines(feature_list, title_list, fname, colours, buckets, plot_hist,
main_title_prefix, x1, x2, xlabel, ylabel, obwiednia=False):
'''
Plot a number of histograms on the same figure.
plot_hist - function responsible for drawing a single histogram, which is passed
list of values, colour, bins list
'''
print "[plot_hists_lines]"
bins = numpy.linspace(min([min(x) for x in feature_list]),
max([max(x) for x in feature_list]), buckets)
from itertools import izip
for feats, col in izip(feature_list, colours):
if obwiednia:
h1,edges1 = numpy.histogram(feats, bins, normed=False)
pyplot.step(edges1[1:], h1, color=col)
else:
plot_hist(feats, col, bins)
full_title = []
if main_title_prefix:
full_title.append(main_title_prefix)
#for title, col in izip(title_list, colours):
# full_title.append(title+" has colour: "+col+" ")
pyplot.title(" ".join(full_title))
#pyplot.axvline(x1, 0, 7000, color="black")
#pyplot.axvline(x2, 0, 7000, color="black")
frame1 = pyplot.gca()
#frame1.get_yaxis().set_visible(False)
pyplot.xlabel(xlabel)
pyplot.ylabel(ylabel)
pyplot.savefig(fname)
pyplot.close()
def plotNimRow(self, n, directory = ''):
'''
Displays the NIM row in a plot.
If self.nims is empty calls self.nimRow()
If value of directory is not set, uses ./plots/<self.mode>-<self.ends>/
'''
if len(directory) < 1:
directory = './plots/' + str(self.mode) + '-' + str(self.ends) + '/'
if not os.path.exists(directory):
try:
os.makedirs(directory)
except Exception as e:
print("Error: " + str(e))
filename = directory + 'plt-d' + str(self.denums).replace('[','').replace(']','').replace(' ','').replace(',',':') + str(self.mode) + str(self.ends) + '-n=' + str(n) + ''
if len(self.nims) < 1:
self.nimRow(n)
plt.clf()
plt.step(range(len(self.nims)), self.nims, 'g', linewidth=4, where='post', fillstyle='full')
plt.xscale('log')
plt.ylim(0, max(self.nims) + 0.5)
plt.xlabel('$n$')
plt.ylabel('$SG(n)$')
plt.savefig(filename + '.ps')
os.system('ps2pdfwr ' + filename + '.ps ' + filename + '.pdf')
os.system('rm ' + filename + '.ps')
os.system('pdfcrop ' + filename + '.pdf ' + filename + '.pdf')
os.system('nohup evince ' + filename + '.pdf > /dev/null &')
def _plot_completeness(comw, start_time, end_time):
comp = np.column_stack([np.hstack([end_time, comw[:, 0], start_time]),
np.hstack([comw[0, 1], comw[:, 1], comw[-1, 1]])])
for row in comp:
print(row[0], row[1])
plt.step(comp[:-1, 0], comp[1:, 1], ls='--',
where="post", linewidth=3, color='brown')
def plotVectors(json, coincidence=0, name="", minTime=30, maxTime=1700,
labels=False):
delims = readDelims()
macAddresses = {}
json["packets"].apply(lambda row: addPacket(row, macAddresses), axis=1)
sum = np.zeros(json["last"], dtype=np.int)
for val in macAddresses.values():
diff = val[1] - val[0]
if ((diff > minTime) and (diff < maxTime) and (val[2] >= coincidence)):
sum[val[0]: val[1]] += 1
plot.xlabel('Seconds since start')
plot.ylabel('Devices', color='b')
plot.step(range(int(json["last"])), sum.tolist())
actual = [stop['actual'] for stop in delims]
stopx = [stop['start'] for stop in delims]
ax2 = plot.twinx()
ax2.step(stopx, actual, 'r', where="post")
(t1.set_color for t1 in ax2.get_yticklabels())
ax2.set_ylabel('Riders', color='r')
plot.ylim(0, 30)
plot.xlim(0, json["last"])
if(labels):
for stop in delims:
annotate(stop["code"], stop["start"], stop["actual"], 10, 10)
makeWidePlot("bus", "vectors")
plot.show()
def fvc_plot_setup(hist_data, hist, binEdges, xlabel, title = ""):
'''
Plot the histogram, with removed observations highlighted
:param array hist_data: raw values which have been binned to create hist
:param array hist: values of histogram
:param array binEdges: location of LH bin edge
:param str xlabel: label for x-axis
:param str title: title of plot
:returns:
plot-hist - useful histogram data to plot in log-scale
bincenters - locations of centres of bins
'''
import matplotlib.pyplot as plt
plot_hist = np.array([float(x) if x != 0 else 1e-1 for x in hist])
plt.clf()
bincenters = 0.5 * (binEdges[1:] + binEdges[:-1])
plt.step(bincenters, plot_hist, 'b-', label = 'observations', where='mid')
fit = utils.fit_gaussian(bincenters, hist, max(hist), mu = np.mean(hist_data), sig = np.std(hist_data))
plot_gaussian = utils.gaussian(bincenters, fit)
plt.plot(bincenters, plot_gaussian, 'r-', label = 'Gaussian fit')
# sort labels and prettify
plt.xlabel(xlabel)
plt.ylabel("Frequency")
plt.gca().set_yscale('log')
plt.ylim([0.1,10000])
plt.title(title)
return plot_hist, bincenters # fvc_plot_setup
def cdf_compare(dists, title, xl, xr, yl, yr, labels):
mm = min(dists[0])
ma = max(dists[0])
cnt = len(dists[0])
for i in xrange(1, len(dists)):
mm = min(mm, min(dists[i]))
ma = max(ma, max(dists[i]))
cnt = min(cnt, len(dists[i]))
print "cnt = " + str(cnt)
x = np.linspace(mm, ma, cnt)
i = 0
for dist in dists:
ecdf = sm.distributions.ECDF(dist)
plt.step(x, ecdf(x), label=labels[i])
i += 1
dist.sort()
# print dist
plt.title(title)
plt.xticks(xr)
plt.yticks(yr)
plt.xlabel(xl)
plt.ylabel(yl)
plt.legend(labels, loc=4)
plt.show()
def plot_pdf(n): w = 0.5 r = 0.9 X = np.random.uniform(0,1,n) Y = map(lambda x: w*x + (1-w)*r, X) # CDF estimate of X plt.figure() ecdf_x = sm.tools.ECDF(X) x = np.linspace(min(X), max(X)) f_x = ecdf_x(x) plt.step(x, f_x) plt.xlabel(r"$x$") plt.ylabel(r"Empirical CDF of $X \sim U(0,1)$") plt.grid() # PDF estimate of Y plt.figure() count, bins, patches = plt.hist(Y, n/200, normed=1) # CDF estimate of Y plt.figure() ecdf_y = sm.tools.ECDF(Y) y = np.linspace(min(Y), max(Y)) f_y = ecdf_y(y) plt.step(y, f_y) plt.xlabel(r"$y$") plt.ylabel(r"Empirical CDF of $Y=w\cdot X + (1-w)\cdot r$") plt.grid()
def pltLumFun(data,lumbins,color='blue',linestyle='-',redshift=1,overplot=False,plotdata=True,label=None,linewidth=2):
zz, = np.where(z==redshift)
plt.step(lumbins,np.log10(np.append(data[zz,:],data[zz,-1])),color=color,linestyle=linestyle,label=label,lw=linewidth)
if plotdata==True:
obs = readcol.readcol('/nobackupp8/mtremmel/DATA/QSOdata/bol_lf_point_dump.dat',twod=False,asdict=True,skipline=38)
obs2 = readcol.readcol('/nobackupp8/mtremmel/DATA/QSOdata/M1450z5_McGreer13.dat',twod=False,asdict=True,skipline=1)
tt, = np.where(obs['redshift']==redshift)
plt.errorbar(obs['lbol'][tt] + loglbol_sun, obs['dphi'][tt],yerr=obs['sig'][tt],fmt='o',color='grey',ecolor='grey',label='Hopkins+ 2007 (Compilation)')
if z[zz] == 6:
plt.errorbar([logLbol6B],[logphi6B],xerr=errlogLbol6B,yerr=errlogphi6B,fmt='^',color='k',label='Barger+2003')
plt.errorbar([logLbol6F],[logphi6F],xerr=[[logLbol6Fm],[logLbol6Fp]],yerr=[[errlogphi6Fm],[errlogphi6Fp]],fmt='s',color='k',label='Fiore+ 2012')
if z[zz] == 5:
l1450 = np.log10(4.4)+mcABconv(obs2['M1450'],c/(0.145e-4))
dphi = 10**obs2['logphi']
dphip = (2./5.) * (dphi+obs2['sig'])
dphim = (2./5.) * (dphi - obs2['sig'])
dphi = np.log10((2./5.)*dphi)
dphierr = [dphi-np.log10(dphim),np.log10(dphip)-dphi]
plt.errorbar(l1450,dphi,yerr=dphierr,fmt='D',color='k',label='McGreer+ 2013')
if overplot==False:
plt.title(str(zbinsl[zz[0]])+' < z < '+str(zbinsh[zz[0]]))
plt.xlabel(r'log$_{10}$($L_{bol}$ [ergs/s]))',fontsize=30)
plt.ylabel(r'log$_{10}$($\phi$ [Mpc$^{-3}$ dex$^{-1}$])',fontsize=30)
plt.legend(loc='lower left',fontsize=20)
return
def hist_alarms(self,alarms,title_str='alarms',save_figure=False,linestyle='-'):
fontsize=15
T_min_warn = self.T_min_warn
T_max_warn = self.T_max_warn
if len(alarms) > 0:
alarms = alarms / 1000.0
alarms = np.sort(alarms)
T_min_warn /= 1000.0
T_max_warn /= 1000.0
plt.figure()
alarms += 0.0001
bins=np.logspace(np.log10(min(alarms)),np.log10(max(alarms)),40)
#bins=linspace(min(alarms),max(alarms),100)
# hist(alarms,bins=bins,alpha=1.0,histtype='step',normed=True,log=False,cumulative=-1)
#
plt.step(np.concatenate((alarms[::-1], alarms[[0]])), 1.0*np.arange(alarms.size+1)/(alarms.size),linestyle=linestyle,linewidth=1.5)
plt.gca().set_xscale('log')
plt.axvline(T_min_warn,color='r',linewidth=0.5)
#if T_max_warn < np.max(alarms):
# plt.axvline(T_max_warn,color='r',linewidth=0.5)
plt.xlabel('Time to disruption [s]',size=fontsize)
plt.ylabel('Fraction of detected disruptions',size=fontsize)
plt.xlim([1e-4,4e1])#max(alarms)*10])
plt.ylim([0,1])
plt.grid()
plt.title(title_str)
plt.setp(plt.gca().get_yticklabels(),fontsize=fontsize)
plt.setp(plt.gca().get_xticklabels(),fontsize=fontsize)
plt.show()
if save_figure:
plt.savefig('accum_disruptions.png',dpi=200,bbox_inches='tight')
else:
print(title_str + ": No alarms!")
def run(self):
dataset = [[float(entry) for entry in data]
for data in self._get_stripped_file_lines()]
if not data:
return None
# Plot aesthetics.
fig = plt.figure(1)
plt.title('%s' % FLAGS.title)
xlabel = FLAGS.xlabel
if FLAGS.xlog:
xlabel = 'log( ' + xlabel + ' )'
plt.xlabel(xlabel)
plt.ylabel(FLAGS.ylabel)
data_plts = []
for i, data in enumerate(dataset):
ecdf = distributions.ECDF(data)
if FLAGS.xmax:
x = np.linspace(0, float(FLAGS.xmax), num=len(data))
else:
x = np.linspace(min(data), max(data), num=len(data))
y = ecdf(x)
plt.step(x, y, '.-', label=self.filepaths[i])
xmin, xmax, ymin, ymax = plt.axis()
plt.axis((xmin, xmax, 0, 1))
plt.legend(loc='lower right')
if FLAGS.xlog:
plt.xscale('log')
fig.savefig(FLAGS.plot_name + '.png')
def PlotStep(self, Primal,Primal0,col,dt = 0.2):
Nshooting = self.Nshooting
Nturbine = self.Nturbine
time = {'States': [dt*k for k in range(Nshooting+1)],
'Inputs': [dt*k for k in range(Nshooting)]}
Nsubp = np.ceil(np.sqrt(Nturbine))
key_dic = {'States': ['Og','beta'], 'Inputs': ['Tg']}
counter = 0
for type_ in key_dic.keys():
for key in key_dic[type_]:
for k in range(Nturbine):
Primal_struc = self._TurbineV(Primal['Turbine'][k])
Primal0_struc = self._TurbineV(Primal0['Turbine'][k])
diff = veccat(Primal_struc[type_,:,key])-veccat(Primal0_struc[type_,:,key])
plt.figure(10+counter)
plt.subplot(Nsubp,Nsubp,k)
plt.hold('on')
if (type_ == 'States'):
plt.plot(time[type_],diff,color=col)
else:
plt.step(time[type_],diff,color=col)
plt.title('Step'+key)
counter += 1
def coc_set_up_plot(bincenters, hist, gaussian, variable, threshold = 0, sub_par = ""):
'''
Set up the plotting space for the Climatological Outlier Check
:param array bincenters: bin centres of histogram
:param array hist: histogram values
:param array gaussian: parameters of gaussian fit [m, s, n]
:param str variable: name of variable for title
:param int threshold: threshold to plot
:param str sub_par: sub parameter for axis label
'''
import matplotlib.pyplot as plt
plt.clf()
plt.axes([0.1,0.15,0.85,0.75])
plot_hist = np.array([0.01 if h == 0 else h for h in hist])
plt.step(bincenters, plot_hist, 'k-', label = 'standardised months', where='mid')
# plot fitted Gaussian
plot_gaussian = utils.gaussian(bincenters, gaussian)
plt.plot(bincenters, plot_gaussian, 'b-', label = 'Gaussian fit')
# sort the labels etc
plt.xlabel("%s offset (IQR)" % variable)
plt.ylabel("Frequency (%s)" % sub_par)
plt.gca().set_yscale('log')
plt.axvline(-threshold-1,c='r')
plt.axvline(threshold+1,c='r')
plt.axvline(-threshold,c='orange')
plt.axvline(threshold,c='orange')
plt.ylim(ymin=0.1)
plt.title("Climatological Gap Check - %s - %s" % (sub_par, variable) )
return # coc_set_up_plot
def dopsd(nfft = None, rpt = 10, plotdB=True):
"""
Takes a snapshot, then computes, plots and writes out the Power Spectral
Density functions. The psd function is written into a file named "psd".
This file will be overwritten with each call. Arguments:
nfft The number of points in the psd function. Defaults to the number of
points in a snapshot, the maximum which should be used.
rpt The numper of mesurements to be averaged for the plot and output file.
Defaults to 10.
plotdB controls whether the plot is linear in power or in dB
"""
if nfft == None:
nfft = numpoints
for i in range(rpt):
power, freqs = adc5g.get_psd(roach2, snap_name, samp_freq*1e6, 8, nfft)
if i == 0:
sp = power
else:
sp += power
sp /= rpt
if plotdB:
plt.step(freqs, 10*np.log10(sp))
else:
plt.step(freqs, (sp))
plt.show(block = False)
data = np.column_stack((freqs/1e6, 10*np.log10(sp)))
np.savetxt("psd", data, fmt=('%7.2f', '%6.1f'))
with open('tokens.json', 'w') as f:
json.dump(data, f)
for user in tokens['users'].values():
client = fitbit.Fitbit(tokens['client_id'],
tokens['client_secret'],
access_token=user['access_token'],
refresh_token=user['refresh_token'],
expires_at=user['expires_at'],
refresh_cb=update_tokens)
client.API_VERSION = 1.2
if len(sys.argv) > 1:
date = sys.argv[1]
else:
date = None
sleep = client.sleep(date=date)
for period in sleep['sleep']:
df = pd.DataFrame(period['levels']['data'])
df.dateTime = pd.to_datetime(df.dateTime)
#df.seconds = pd.to_timedelta(df.seconds, unit='s')
df['levelnr'] = df.level.map(levelnr)
df = df.set_index('dateTime')
#breakpoint()
print(df)
#df.level.plot()
plt.step(df.index.to_list(), df.levelnr.to_list(), where='post')
plt.yticks(list(levelnr.values()), list(levelnr.keys()))
plt.show()
import matplotlib
matplotlib.use('Agg')
from balsam.core import models
from matplotlib import pyplot as plt
import matplotlib.dates as mdates
fig, ax = plt.subplots()
plt.title('Balsam: Tasks in waiting state v Date/Time')
plt.xlabel('Time of Day (H:M)')
plt.ylabel('Num. Tasks Waiting')
times, waiting = models.job_waiting_report()
plt.step(times, waiting, where='post')
myFmt = mdates.DateFormatter('%H:%M')
ax.xaxis.set_major_formatter(myFmt)
fig.autofmt_xdate()
plt.savefig('balsam_waiting_v_time.png')
for j in range(nx):
simX[i,j] = x0[j]
for j in range(nu):
simU[i,j] = u0[j]
# update initial condition
x0 = acados_solver.get(1, "x")
acados_solver.set(0, "lbx", x0)
acados_solver.set(0, "ubx", x0)
# plot results
import matplotlib
import matplotlib.pyplot as plt
t = np.linspace(0.0, Tf/N, Nsim)
plt.subplot(2, 1, 1)
plt.step(t, simU, color='r')
plt.title('closed-loop simulation')
plt.ylabel('u')
plt.xlabel('t')
plt.grid(True)
plt.subplot(2, 1, 2)
plt.plot(t, simX[:,1])
plt.ylabel('theta')
plt.xlabel('t')
plt.grid(True)
plt.show()
def main():
# (a) Input
code = ca_code.generate(prn=1, repeats=10)
# rx = np.roll(ca_code.generate(prn=1, repeats=10), -10230/6*1) | ca_code.generate(prn=2, repeats=10)
rx = np.roll(ca_code.generate(prn=1, repeats=10), 1000) | ca_code.generate(
prn=2, repeats=10)
noise_std = 1
noise = np.random.normal(
scale=noise_std,
size=code.size,
)
rx = np.add(rx, noise)
# (b) Aliasing
q = 2
code_aliased = sfft_aliasing.execute(code, q)
rx_aliased = sfft_aliasing.execute(rx, q)
# (c) FFT
code_f = fft(code_aliased)
rx_f = fft(rx_aliased)
# (d) Multiply
code_rx_f = np.multiply(np.conjugate(code_f), rx_f)
# (e) IFFT
code_rx = np.real(ifft(code_rx_f))
print 'arg max is', np.argmax(code_rx)
plt.figure('Local C/A code')
plt.step(np.arange(code.size), code)
plt.xlim(0, code.size - 1)
plt.title('Local C/A code')
plt.xlabel('Time')
plt.setp(plt.gca().get_xticklabels(), visible=False)
plt.setp(plt.gca().get_yticklabels(), visible=False)
plt.figure('Received signal')
plt.plot(rx)
plt.xlim(0, code.size - 1)
plt.title('Received signal')
plt.xlabel('Time')
plt.setp(plt.gca().get_xticklabels(), visible=False)
plt.setp(plt.gca().get_yticklabels(), visible=False)
plt.figure('Aliased Local C/A code')
plt.step(np.arange(code_aliased.size), code_aliased)
plt.xlim(0, code_aliased.size - 1)
plt.title('Local C/A code (Aliased)')
plt.xlabel('Time')
plt.setp(plt.gca().get_xticklabels(), visible=False)
plt.setp(plt.gca().get_yticklabels(), visible=False)
plt.figure('Aliased Received signal')
plt.plot(rx_aliased)
plt.xlim(0, rx_aliased.size - 1)
plt.title('Received signal (Aliased)')
plt.xlabel('Time')
plt.setp(plt.gca().get_xticklabels(), visible=False)
plt.setp(plt.gca().get_yticklabels(), visible=False)
plt.figure('FFT of Local CA code')
plt.plot(np.abs(fftshift(code_f)))
plt.title('FFT of Local C/A code')
plt.xlabel('Frequency')
plt.xlim(1000, 4000)
plt.ylim(0, 500)
plt.setp(plt.gca().get_xticklabels(), visible=False)
plt.setp(plt.gca().get_yticklabels(), visible=False)
plt.figure('FFT of Received signal')
plt.plot(np.abs(fftshift(rx_f)))
plt.title('FFT of Received signal')
plt.xlabel('Frequency')
plt.xlim(1000, 4000)
plt.ylim(0, 500)
plt.setp(plt.gca().get_xticklabels(), visible=False)
plt.setp(plt.gca().get_yticklabels(), visible=False)
plt.figure('Multiply')
plt.plot(np.abs(fftshift(code_rx_f)))
plt.title('Multiply')
plt.xlabel('Frequency')
plt.xlim(1500, 3500)
plt.ylim(0, 100000)
plt.setp(plt.gca().get_xticklabels(), visible=False)
plt.setp(plt.gca().get_yticklabels(), visible=False)
plt.figure('IFFT')
plt.plot(code_rx)
plt.title('IFFT')
plt.xlim(0, code_rx.size - 1)
plt.setp(plt.gca().get_xticklabels(), visible=False)
plt.setp(plt.gca().get_yticklabels(), visible=False)
plt.show()
)] + non_diabetic[0:int(0.66 * len(non_diabetic))]
test_data = diabetic[int(0.66 * len(diabetic)):len(diabetic)] + non_diabetic[
int(0.66 * len(non_diabetic)):len(non_diabetic)]
_train_d, _train_l = zip(*train_data)
_test_d, _test_l = zip(*test_data)
_train_d = list(_train_d)
_test_d = list(_test_d)
# plot some lame estimator stuff
_event, _time = split_for_kaplan(_train_l, _train_d, 2)
for i in range(0, len(_event)):
x, y = kaplan_meier_estimator(_event[i], _time[i])
plt.step(x, y, where="post", label=str(i))
plt.legend()
plt.plot()
_train_l = numpy.array(list(_train_l), dtype='bool,f4')
_test_l = numpy.array(list(_test_l), dtype='bool,f4')
# create ph model
estimator = CoxPHSurvivalAnalysis()
estimator.fit(_train_d, _train_l)
# create the cox model
plt.text(185, 1.05, "ToO")
plt.annotate('', xy=(0.17, 1.03), xytext=(0.5,1.03), xycoords='axes fraction',
arrowprops=dict(facecolor='black', arrowstyle='<->'),
)
ax1.set_xlim([15,600])
ax1.set_ylim([0,1.0])
ax2.set_ylim([0,1.0])
plotName = os.path.join(outputDir,'efficiency.pdf')
plt.savefig(plotName,bbox_inches='tight')
plt.close()
plt.figure(figsize=(10,6))
for lcurve in data[event].keys():
if data[event][lcurve]["ntiles"].size == 0: continue
plt.step(data[event][lcurve]["ntiles"][:,0], data[event][lcurve]["ntiles"][:,1], '.-', where='mid',label=data[event][lcurve]["label"])
plt.legend(loc=3)
plotName = os.path.join(outputDir,'ntiles.pdf')
plt.savefig(plotName,bbox_inches='tight')
plt.close()
plt.figure(figsize=(10,6))
for lcurve in data[event].keys():
if data[event][lcurve]["probs"].size == 0: continue
plt.step(data[event][lcurve]["probs"][:,0], data[event][lcurve]["probs"][:,1], 'x', label=data[event][lcurve]["label"])
plt.legend(loc=3)
plotName = os.path.join(outputDir,'probs.pdf')
plt.savefig(plotName,bbox_inches='tight')
plt.close()
def get_DR_polar():
# for polar
DR_p1 = []
DR_p2 = []
cts_bin = np.linspace(75, 3575, 36)
cts_bin = np.concatenate(([35], cts_bin))
# cts_bin = np.concatenate((cts_bin,[1000,1500,1800]))
path_p1 = '/Users/baotong/Desktop/period_LW/simulation/simulation_LW_5540/'
path_p2 = '/Users/baotong/Desktop/period_LW/simulation/simulation_LW_45540/'
detect_rate_p1 = sim_sin.get_info(path_p1, 5540.)[0]
detect_rate_p2 = sim_sin.get_info(path_p2, 45540.)[0]
for i in range(len(detect_rate_p1)):
DR_p1.append(np.mean(detect_rate_p1[i]))
DR_p2.append(np.mean(detect_rate_p2[i]))
DR_p1.extend(np.zeros(30) + 1.)
DR_p2.extend(np.zeros(30) + 1.)
print(DR_p1)
print(DR_p2)
cts_hist = plt.hist(cts, histtype='step', bins=cts_bin)[0] * 0.9
plt.close()
alpha = 0.18 #for fraction
beta = 0.25 # ??? for polar intrinsic
det_n1 = cts_hist * DR_p1 * 0.82 * alpha * beta
det_n2 = cts_hist * DR_p1 * 0.18 * alpha * beta
de_sin_polar = np.sum(det_n1[1:]) + np.sum(det_n2[1:])
Lum_LW = np.array([
31.26, 32.36, 31.82, 32.35, 31.75, 31.97, 31.83, 31.25, 31.54, 31.33,
31.98, 31.88, 31.93, 32.11, 31.93, 32.05, 31.62, 32.92, 31.73, 32.78,
31.97, 31.94, 30.88
])
cts_LW = np.array([
202, 902, 394, 784, 293, 437, 335, 121, 347, 211, 438, 760, 512, 823,
487, 535, 214, 3402, 263, 1963, 307, 1039, 138
])
cts_LW_ex_IP = np.array([
902, 394, 784, 293, 437, 335, 121, 347, 211, 438, 760, 512, 823, 487,
535, 214, 3402, 263, 307
])
plt.scatter(cts_LW, Lum_LW)
# plt.hist(det_n1+det_n2,bins=20,histtype = 'step')
DET = det_n1 + det_n2
DET = np.concatenate((DET, [0.]))
a = plt.hist(cts_LW_ex_IP, bins=cts_bin, histtype='step')
plt.close()
plt.figure(1, (8, 6))
font1 = {
'family': 'Normal',
'weight': 'normal',
'size': 15,
}
plt.ylim(0., 4.5)
plt.tick_params(labelsize=15)
plt.step(cts_bin, DET, where='post')
plt.step(cts_bin, np.concatenate((a[0], [0.])), where='post')
plt.legend(['predicted', 'detected'])
plt.xlabel('Counts', font1)
plt.ylabel('Number of sources', font1)
plt.savefig(
'/Users/baotong/Desktop/aas/mod_MN_pCV/figure/sim_LW/est_obs.eps')
plt.show()
print(de_sin_polar)
return de_sin_polar
def plot_pedestals(data_file,
pedestal_file,
run=0,
plot_file=None,
tel_id=1,
offset_value=400,
sample_size=1000):
"""
plot pedestal quantities quantities
Parameters
----------
data_file: pedestal run
pedestal_file: file with drs4 corrections
run: run number of data to be corrected
plot_file: name of output pdf file
tel_id: id of the telescope
offset_value: baseline off_set
"""
config = {
"LSTEventSource": {
"pointing_information": False,
"allowed_tels": [1],
"LSTR0Corrections": {
"drs4_pedestal_path": pedestal_file,
},
}
}
# event_reader
reader = EventSource(data_file, config=Config(config), max_events=None)
t = np.linspace(2, 37, 36)
# configuration for the charge integrator
charge_config = Config({
"FixedWindowSum": {
"window_shift": 6,
"window_width": 12,
"peak_index": 18,
}
})
# declare the pedestal component
pedestal = PedestalIntegrator(
tel_id=tel_id,
time_sampling_correction_path=None,
sample_size=sample_size,
sample_duration=1000000,
charge_median_cut_outliers=[-10, 10],
charge_std_cut_outliers=[-10, 10],
charge_product="FixedWindowSum",
config=charge_config,
subarray=reader.subarray,
)
for i, event in enumerate(reader):
if tel_id != event.trigger.tels_with_trigger[0]:
raise Exception(
f"Given wrong telescope id {tel_id}, files has id {event.trigger.tels_with_trigger[0]}"
)
are_pedestals_calculated = pedestal.calculate_pedestals(event)
if are_pedestals_calculated:
ped_data = event.mon.tel[tel_id].pedestal
break
camera_geometry = reader.subarray.tels[tel_id].camera.geometry
camera_geometry = camera_geometry.transform_to(EngineeringCameraFrame())
if are_pedestals_calculated and plot_file is not None:
with PdfPages(plot_file) as pdf:
plt.rc("font", size=15)
# first figure
fig = plt.figure(1, figsize=(12, 24))
plt.tight_layout()
n_samples = charge_config["FixedWindowSum"]["window_width"]
fig.suptitle(f"Run {run}, integration on {n_samples} samples",
fontsize=25)
pad = 420
image = ped_data.charge_median
mask = ped_data.charge_median_outliers
for chan in np.arange(2):
pad += 1
plt.subplot(pad)
plt.tight_layout()
disp = CameraDisplay(camera_geometry)
mymin = np.median(image[chan]) - 2 * np.std(image[chan])
mymax = np.median(image[chan]) + 2 * np.std(image[chan])
disp.set_limits_minmax(mymin, mymax)
disp.highlight_pixels(mask[chan], linewidth=2)
disp.image = image[chan]
disp.cmap = plt.cm.coolwarm
# disp.axes.text(lposx, 0, f'{channel[chan]} pedestal [ADC]', rotation=90)
plt.title(f"{channel[chan]} pedestal [ADC]")
disp.add_colorbar()
image = ped_data.charge_std
mask = ped_data.charge_std_outliers
for chan in np.arange(2):
pad += 1
plt.subplot(pad)
plt.tight_layout()
disp = CameraDisplay(camera_geometry)
mymin = np.median(image[chan]) - 2 * np.std(image[chan])
mymax = np.median(image[chan]) + 2 * np.std(image[chan])
disp.set_limits_minmax(mymin, mymax)
disp.highlight_pixels(mask[chan], linewidth=2)
disp.image = image[chan]
disp.cmap = plt.cm.coolwarm
# disp.axes.text(lposx, 0, f'{channel[chan]} pedestal std [ADC]', rotation=90)
plt.title(f"{channel[chan]} pedestal std [ADC]")
disp.add_colorbar()
# histograms
for chan in np.arange(2):
mean_ped = ped_data.charge_mean[chan]
ped_std = ped_data.charge_std[chan]
# select good pixels
select = np.logical_not(mask[chan])
# fig.suptitle(f"Run {run} channel: {channel[chan]}", fontsize=25)
pad += 1
# pedestal charge
plt.subplot(pad)
plt.tight_layout()
plt.ylabel("pixels")
plt.xlabel(f"{channel[chan]} pedestal")
median = np.median(mean_ped[select])
rms = np.std(mean_ped[select])
label = f"{channel[chan]} Median {median:3.2f}, std {rms:3.2f}"
plt.hist(mean_ped[select], bins=50, label=label)
plt.legend()
pad += 1
# pedestal std
plt.subplot(pad)
plt.ylabel("pixels")
plt.xlabel(f"{channel[chan]} pedestal std")
median = np.median(ped_std[select])
rms = np.std(ped_std[select])
label = f" Median {median:3.2f}, std {rms:3.2f}"
plt.hist(ped_std[select], bins=50, label=label)
plt.legend()
plt.subplots_adjust(top=0.94, bottom=0.04, right=0.96)
pdf.savefig()
plt.close()
# event_reader
# reader = EventSource(data_file, config=Config(config), max_events=1000)
pix = 0
pad = 420
# plot corrected waveforms of first 8 events
for i, ev in enumerate(reader):
for chan in np.arange(2):
if pad == 420:
# new figure
fig = plt.figure(ev.index.event_id * 1000,
figsize=(12, 24))
fig.suptitle(f"Run {run}, pixel {pix}", fontsize=25)
plt.tight_layout()
pad += 1
plt.subplot(pad)
# remove samples at beginning / end of waveform
start = reader.r0_r1_calibrator.r1_sample_start.tel[tel_id]
end = reader.r0_r1_calibrator.r1_sample_end.tel[tel_id]
plt.subplots_adjust(top=0.92)
label = f"event {ev.index.event_id}, {channel[chan]}: R0"
plt.step(
t,
ev.r0.tel[tel_id].waveform[chan, pix, start:end],
color="blue",
label=label,
)
label = "baseline correction \n + dt corr + corrected spikes"
plt.step(
t,
ev.r1.tel[tel_id].waveform[chan, pix] + offset_value,
alpha=0.5,
color="green",
label=label,
)
plt.plot([0, 40], [offset_value, offset_value],
"k--",
label="offset")
plt.xlabel("time sample [ns]")
plt.ylabel("counts [ADC]")
plt.legend()
plt.ylim(200, 600)
if pad == 428:
pad = 420
plt.subplots_adjust(top=0.92)
pdf.savefig()
plt.close()
if i == 8:
break
elif not are_pedestals_calculated:
log.error(
"Not able to calculate pedestals or output pdf file not especified."
)
elif plot_file is None:
log.warning("Not PDF outputfile specified.")
def get_ml_toa(fits_fn,
prof_mod,
parfile,
scope='swift',
print_offs=None,
frequency=None,
fdot=None,
epoch=None,
sim=False,
bg_counts=0,
Emin=None,
Emax=None,
gauss_err=False,
tempo2=False,
debug=False,
split_n_days=None,
correct_pf=False,
split_num=None,
split_orbits=False,
split_photons=False,
writefile=False,
bright=False):
print_timings = False # if want to print summary of runtime
if split_n_days == None and split_photons == None:
fits = pyfits.open(fits_fn)
t = smu.fits2times(fits_fn, scope=scope, Emin=Emin, Emax=Emax)
#if scope != 'chandra':
# exposure = fits[0].header['EXPOSURE']
try:
obsid = fits[0].header['OBS_ID']
except KeyError:
obsid = os.path.basename(fits_fn)
else:
obsid = 'split_' + str(split_n_days) + '_days'
if bg_counts < 0:
bg_scale = -1.0 * bg_counts
bg_fits_fn = fits_fn.replace('reg', 'bgreg')
bg_fits = pyfits.open(bg_fits_fn)
bg_counts = int(bg_fits[1].header['NAXIS2'] * bg_scale)
print 'BG Counts:', bg_counts
bg_fits.close()
if frequency and epoch:
par = lambda: None
par.epoch = epoch
par.f0 = frequency
par.fdots = [fdot, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]
else:
par = PSRpar(parfile)
# split times into multiple arrays if needed
if split_orbits:
dt = t[1:] - t[:-1]
splits = np.where(dt > 0.0116)[0] # 1 ks in days
if len(splits):
ts = [t[:splits[0]]]
for i in range(len(splits) - 1):
ts.append(t[splits[i] + 1:splits[i + 1]])
ts.append(t[splits[-1] + 1:])
else:
ts = np.atleast_2d(t)
elif split_num:
remainder = len(t) % split_num
if remainder:
sys.stderr.write("Warning: Number of events in %s not divisable by %d. " \
"Dropping last %d events.\n" % (obsid, split_num, remainder))
ts = np.split(t[:-remainder], split_num)
else:
ts = np.split(t, split_num)
elif split_photons:
t = np.zeros(0)
fits_files = np.loadtxt(fits_fn, dtype='S')
try:
for fit in fits_files:
t = np.append(
t, smu.fits2times(fit, scope=scope, Emin=Emin, Emax=Emax))
except (Exception):
t = np.append(
t,
smu.fits2times(fits_files.tostring(),
scope=scope,
Emin=Emin,
Emax=Emax))
t.sort()
n_toas = len(t) / split_photons
if n_toas == 0:
ts = np.split(t, 1)
else:
remainder = len(t) % n_toas
if remainder:
sys.stderr.write("Warning: Number of events in %s not divisable by %d. " \
"Dropping last %d events.\n" % (obsid, n_toas, remainder))
ts = np.split(t[:-remainder], n_toas)
else:
ts = np.split(t, n_toas)
elif split_n_days:
t = np.zeros(0)
fits_files = np.loadtxt(fits_fn, dtype='S')
try:
for fit in fits_files:
t = np.append(
t, smu.fits2times(fit, scope=scope, Emin=Emin, Emax=Emax))
except (Exception):
t = np.append(
t,
smu.fits2times(fits_files.tostring(),
scope=scope,
Emin=Emin,
Emax=Emax))
t.sort()
dt = t[1:] - t[:-1]
splits = np.where(dt > 0.0116)[0] # 1 ks in days 0.0116
newsplit = []
gaps = t[1:][splits] - t[0]
while (gaps[-1] > split_n_days):
newsplit.append(splits[np.where(gaps > split_n_days)[0][0]])
gaps = gaps - gaps[np.where(gaps > split_n_days)[0][0]]
ts = np.split(t, np.array(newsplit) + 1)
else:
ts = np.atleast_2d(t)
if len(ts) > 1 and debug:
plt.figure()
for t in ts:
nbins = int((t[-1] - t[0]) * 8640.0)
hist = np.histogram(t, bins=nbins)
plt.plot(hist[1][:-1], hist[0], c='b')
plt.axvline(t[0], ls='--', c='k', lw=2)
plt.axvline(t[-1], ls='-', c='k', lw=2)
plt.show()
for i, t in enumerate(ts):
if split_n_days:
sys.stderr.write('Measuring TOA #%d of %d\n' % (i + 1, len(ts)))
else:
sys.stderr.write('Measuring TOA #%d for %s\n' % (i + 1, obsid))
phases = smu.times2phases(t, parfile)
if correct_pf:
old_model, new_model, corr_folded = correct_model(phases, prof_mod)
maxoff, error = calc_toa_offset(phases,
prof_mod.prof_mod,
sim_err=sim,
bg_counts=bg_counts,
gauss_err=gauss_err,
debug=debug,
bright=bright)
midtime = (t[-1] + t[0]) / 2.0
p_mid = 1.0 / smu.calc_freq(midtime, par.epoch, par.f0, par.fdots[0],
par.fdots[1], par.fdots[2], par.fdots[3],
par.fdots[4], par.fdots[5], par.fdots[6],
par.fdots[7], par.fdots[8])
t0 = smu.calc_t0(midtime, par.epoch, par.f0, par.fdots[0],
par.fdots[1], par.fdots[2], par.fdots[3],
par.fdots[4], par.fdots[5], par.fdots[6],
par.fdots[7], par.fdots[8])
t0i = int(t0)
t0f = t0 - t0i
toaf = t0f + maxoff * p_mid / SECPERDAY
newdays = int(np.floor(toaf))
if tempo2:
smu.write_tempo2_toa(t0i + newdays,
toaf - newdays,
error * p_mid * 1.0e6,
0000,
0.0,
name=obsid,
writefile=writefile)
else:
smu.write_princeton_toa(t0i + newdays,
toaf - newdays,
error * p_mid * 1.0e6,
0000,
0.0,
name=obsid)
if print_offs:
offs_file = open(print_offs, 'a')
#print "\t",error*p_mid*1.0e6,"\t",exposure # this was for checking uncertainties scaling with exposure time
offs_file.write(fits_fn + "\t" + str(maxoff) + "\t" + str(error) +
"\n")
#print obsid,"\tOffset:",maxoff,"+/-",error
offs_file.close()
if split_n_days == None and split_photons == False:
fits.close()
#double check PF correction with measuring binned model pulsed fraction
if correct_pf and debug:
plt.figure()
nbins = len(corr_folded[0])
uncertainties = np.sqrt(corr_folded[0])
area = np.sum(corr_folded[0], dtype='float') / nbins
plt.step(corr_folded[1][:-1],
np.roll(corr_folded[0] / area, int(1.0 - maxoff * nbins)),
where='mid')
plt.errorbar(corr_folded[1][:-1],
np.roll(corr_folded[0] / area,
int(1.0 - maxoff * nbins)),
uncertainties / area,
fmt='ko')
model_x = np.linspace(0, 1, 100)
plt.plot(model_x, old_model(model_x), label='uncorrected')
plt.plot(model_x, new_model(model_x), label='corrected')
plt.legend()
plt.show()
if print_timings:
global calcprobtime
global logsumtime
global integratetime
sys.stderr.write('\tCalc Prob: %f s\n' % calcprobtime)
sys.stderr.write('\tLog Sum: %f s\n' % logsumtime)
sys.stderr.write('\tIntegrate Norm: %f s\n' % integratetime)
T11 += h11.delay
g12 = h12.tf
T12, y12 = signal.step(g12)
T12 += h12.delay
g21 = h21.tf
T21, y21 = signal.step(g21)
T21 += h21.delay
g22 = h22.tf
T22, y22 = signal.step(g22)
T22 += h22.delay
#%% Plot original vs OPOM
plt.subplot(221).set_title("y11")
plt.plot(T11, y11)
plt.step(t, y[0][:,0])
plt.subplot(222).set_title("y12")
plt.plot(T12, y12)
plt.step(t, y[1][:,0])
plt.subplot(223).set_title("y21")
plt.plot(T21, y21)
plt.step(t, y[0][:,1])
plt.subplot(224).set_title("y22")
plt.plot(T22, y22)
plt.step(t, y[1][:,1])
#%% show
plt.show()
for train_indices, test_indices in splitter.split(X, Y):
X_train, Y_train = X.iloc[train_indices], Y.iloc[train_indices]
X_test, Y_test = X.iloc[test_indices], Y.iloc[test_indices]
X_train = normalize(X_train)
X_test = normalize(X_test)
model.fit(X_train, Y_train)
predicted = model.predict(X_test)
print('\nClassification report: \n{}'.format(
classification_report(Y_test, predicted)))
det, tot1, tot2 = undetectedFraudRate(Y_test.values, predicted)
print('Precision:\t{} %\nRecall:\t\t{} %'.format(
det / tot2 * 100, det / tot1 * 100))
avg_precision = average_precision_score(Y_test, predicted)
precision, recall, _ = precision_recall_curve(Y_test, predicted)
plt.step(recall, precision, color='r', alpha=0.2, where='post')
plt.fill_between(recall, precision, step='post', alpha=0.2, color='r')
plt.xlabel('Recall')
plt.ylabel('Precision')
plt.xlim([0.0, 1.05])
plt.ylim([0.0, 1.0])
plt.title('Fraud detection precision-recall curve: AP = {}'.format(
avg_precision))
print('\nExecution time: {} seconds'.format(time.time() - st_time))
alpha = float(input("Enter alpha lethality coefficient (ex: 0.8): "))
beta = float(input("Enter beta lethality coefficient (ex: 0.9): "))
# Integrate using Euler's method
for i in range(N - 1):
time[i + 1] = time[i] + dt
dxdt = -1 * (beta * y_troops[i])
dydt = -1 * (alpha * x_troops[i])
dx = dxdt * dt
dy = dydt * dt
# When either x_troops or y_troops reaches zero then there are no further casualties
if x_troops[i] <= 0:
x_troops[i] = 0
else:
x_troops[i + 1] = x_troops[i] + dx
if y_troops[i] <= 0:
y_troops[i] = 0
else:
y_troops[i + 1] = y_troops[i] + dy
print("%6.2f %6.2f %6.2f" % (time[i], x_troops[i], y_troops[i]))
# Display graph
plt.figure()
plt.step(time, x_troops, '-b', label='x-type')
plt.step(time, y_troops, '-r', label='y-type')
plt.xlabel('Time', weight='bold')
plt.ylabel('Survivors', weight='bold')
plt.legend(loc='best')
plt.title("Lancaster's Law for Aimed Fire - Euler's Method", weight='bold')
plt.show()
t_projection = np.arange(vic_dates[-1], vic_dates[-1] + days_projection + 1)
SEP = np.datetime64('2021-09-01')
OCT = np.datetime64('2021-10-01')
vic_proj_rate = np.zeros(len(t_projection))
vic_proj_rate[:] = 0.2
# clip to 85% fully vaxed
initial_coverage = 100 * vic_doses.sum() / POPS[STATE]
vic_proj_rate[initial_coverage + vic_proj_rate.cumsum() > 2 * 80.0] = 0
plt.figure(figsize=(10, 5))
plt.subplot(121)
vic_rate = 100 * seven_day_average(vic_doses) / POPS[STATE]
plt.step(vic_dates, vic_rate, label="Actual")
plt.step(t_projection, vic_proj_rate, label="Assumed for projection")
plt.legend()
locator = mdates.DayLocator([1, 15])
formatter = mdates.ConciseDateFormatter(locator)
plt.gca().xaxis.set_major_locator(locator)
plt.gca().xaxis.set_major_formatter(formatter)
plt.grid(True, color='k', linestyle=":", alpha=0.5)
plt.axis(
xmin=np.datetime64('2021-05-01'), xmax=np.datetime64('2021-12-31'), ymax=2.2, ymin=0
)
plt.ylabel('7d avg doses per hundred population per day')
plt.title(f"{STATE} daily vaccinations per capita")
plt.subplot(122)
vic_cumulative = 100 * vic_doses.cumsum() / POPS[STATE]
# Add equality constraint
g += [Xk_end - Xk]
lbg += [0, 0]
ubg += [0, 0]
# Create an NLP solver
prob = {'f': J, 'x': vertcat(*w), 'g': vertcat(*g)}
solver = nlpsol('solver', 'ipopt', prob)
# Solve the NLP
sol = solver(x0=w0, lbx=lbw, ubx=ubw, lbg=lbg, ubg=ubg)
w_opt = sol['x'].full().flatten()
# Plot the solution
x1_opt = w_opt[0::3]
x2_opt = w_opt[1::3]
u_opt = w_opt[2::3]
tgrid = [T / N * k for k in range(N + 1)]
import matplotlib.pyplot as plt
plt.figure(1)
plt.clf()
plt.plot(tgrid, x1_opt, '--')
plt.plot(tgrid, x2_opt, '-')
plt.step(tgrid, vertcat(DM.nan(1), u_opt), '-o')
plt.xlabel('t')
plt.legend(['x1', 'x2', 'u'])
plt.grid()
plt.ylim(-0.3, 0.25)
plt.savefig('1.png', format='png')
plt.show()
np.random.seed(123)
cos = np.array([np.random.normal(.0, .3) for i in range(10000)])
cos0 = [c for c in cos if c >= 0. and c <= 1.]
cos = np.array([np.random.normal(1., .3) for i in range(10000)])
cos1 = [c for c in cos if c >= 0. and c <= 1.]
plt.hist(cos0, color='b', alpha=.8)
plt.hist(cos1, color='g', alpha=.8)
plt.show()
cos, y, ecdf, yfit, d_yfit, a2 = fits(cos0)
print('a2=', a2)
ascii.write(Table(np.column_stack([cos, ecdf(cos), y, yfit, d_yfit])),
'../data/fits_ones',
names=['cos', 'ecdf', 'y', 'yfit', 'd_yfit'],
overwrite=True)
plt.step(cos, y, 'b-')
plt.plot(cos, yfit, 'k--')
cos, y, ecdf, yfit, d_yfit, a2 = fits(cos1)
print('a2=', a2)
ascii.write(Table(np.column_stack([cos, ecdf(cos), y, yfit, d_yfit])),
'../data/fits_zeros',
names=['cos', 'ecdf', 'y', 'yfit', 'd_yfit'],
overwrite=True)
plt.step(cos, y, 'g-')
plt.plot(cos, yfit, 'k--')
plt.show()
# %%
# A partir de los autovalores, calculamos la varianza explicada
tot = sum(eig_vals)
print(tot)
var_exp = [(i / tot) * 100 for i in sorted(eig_vals, reverse=True)]
print(var_exp)
cum_var_exp = np.cumsum(var_exp)
print(cum_var_exp)
# Representamos en un diagrama de barras la varianza explicada por cada autovalor, y la acumulada
with plt.style.context('seaborn-pastel'):
plt.figure(figsize=(6, 4))
plt.step(range(4000),
cum_var_exp,
where='mid',
linestyle='--',
label='Varianza explicada acumulada')
plt.ylabel('Ratio de Varianza Explicada')
plt.xlabel('Componentes Principales')
plt.legend(loc='best')
plt.tight_layout()
plt.show()
#----------------------------------------------------------------------------------------------------------------------
#El código a continuación tiene .85 para el parámetro de número de componentes que obtenemos de la grafica anterior debido
#a que el codo de la grafica se produce en el valor 85. Por tanto en estos componentes se muestra cuánta varianza se puede
#atribuir a cada una de estas componentes principales.
#Significa que scikit-learn elige el número mÃnimo de componentes principales de manera que se retenga el 85% de la varianza.
plt.ylim(.475, .535)
plt.ylabel('$C_A$')
plt.plot(time, res1_xk[:, 0], time, bl * .485)
plt.legend(['$C_A$', 'Lower Constraint'])
plt.grid()
plt.savefig(case + '_a.png')
if b:
b = plt.figure(set[1])
plt.clf()
plt.subplot(2, 1, 1)
plt.title(case)
plt.subplot(2, 1, 1)
plt.ylim(35, 125)
plt.ylabel('q')
plt.step(time[0:-1], res1_uk[:, 0], time[0:-1], bl[0:-1] * 40,
time[0:-1], bl[0:-1] * 120)
#plt.legend(['q', 'lower bound', 'upper bound'])
plt.grid()
plt.subplot(2, 1, 2)
plt.ylim(285, 325)
plt.ylabel('$T_c$')
plt.step(time[0:-1], res1_uk[:, 1], time[0:-1], bl[0:-1] * 290,
time[0:-1], bl[0:-1] * 320)
#plt.legend(['$T_c$', 'lower bound', 'upper bound'])
plt.grid()
plt.savefig(case + '_b.png')
if c:
c = plt.figure(set[2])
plt.clf()
plt.subplot(2, 1, 1)
def plot_Fidelity(image2plot, Nplots, Ny, title):
grid = plt.GridSpec(ncols=3, nrows=Nplots, wspace=0.1, hspace=0.2)
# Fidelity plot
xminplot = -1
xmaxplot = 100.
nchans, b, mids, h = get_values(FITSfile=image2plot,
xmin=xminplot,
xmax=xmaxplot,
xstep=1)
h[h == -inf] = np.nan
# 2D plot per channel
ax1 = plt.subplot(grid[Ny, :2])
plt.title(title + " - Fidelity",
fontsize=20,
x=0,
ha="left",
position=(0.1, 0.8))
#plt.title("Q parameter",fontsize=10,x=1,ha="right",color='grey', style='italic')
plt.imshow(h,
extent=(-0.5, nchans - 0.5, xminplot, xmaxplot),
aspect='auto',
vmin=0,
cmap="jet",
interpolation='none',
origin='lower')
#plt.hlines(0,-0.5,nchans-0.5,linestyle="--",color="black",linewidth=3,label="Goal")
plt.xlabel("Channel number", fontsize=15)
plt.ylabel("Fidelity", fontsize=18)
h[np.isnan(h)] = 0.0
cbar = plt.colorbar()
cbar.set_label(r'log$_{10}(\# pixels)$', fontsize=15)
# Histogram
ax2 = plt.subplot(grid[Ny, 2])
plt.hlines(0,
0.1,
max(h.flat),
linestyle="--",
color="black",
linewidth=3,
label="Goal",
alpha=1.,
zorder=-2)
# Get mean values (only orientative)
for i in np.arange(0, nchans):
plt.step(h[:, i], mids, color="grey", linewidth=1, alpha=0.1)
##h[h == 0.0] = np.nan
# We use linear weights rather than log10
plt.step(np.log10(np.average(10.**h, axis=1)),
mids,
color="red",
linewidth=3,
label="mean")
meanvalue = np.round(
np.average(mids, weights=np.log10(np.average(10.**h, axis=1))), 2)
sigmavalue = np.round(
np.sqrt(np.cov(mids, aweights=np.log10(np.average(10.**h, axis=1)))),
2)
# The alternative would be
#plt.step(np.nanmean(h,axis=1)[:],mids,color="red",linewidth=3,label="mean")
#meanvalue = np.round(np.average(mids,weights=np.nanmean(h,axis=1)[:]),2)
#sigmavalue = np.round(np.sqrt(np.cov(mids, aweights=np.nanmean(h,axis=1)[:])),2)
plt.title('{:.2f}'.format(meanvalue) + "+/-" + '{:.2f}'.format(sigmavalue),
fontsize=15,
x=0,
ha="left",
position=(0.6, 0.1),
color="blue")
plt.hlines(meanvalue,
0.1,
max(h.flat),
color="blue",
linewidth=3,
label="average",
alpha=0.5,
zorder=-2)
# Plot parameters
plt.xlim(0, max(h.flat))
plt.ylabel("Fidelity", fontsize=18)
plt.xlabel(r'log$_{10}(\# pixels)$', fontsize=15)
plt.legend(loc="upper left", prop={"size": 10})
ax2.tick_params(labelbottom=True,
labelleft=False,
labelright=True,
bottom=True,
top=True,
left=True,
right=True)
ax2.yaxis.set_label_position("right")
# return for comparisons
return mids, np.nanmean(h, axis=1)[:]
def show_nice_posterior_correlations(post,
x_gt,
prior_hists,
spectrum_number=0,
x_start=0,
x_end=c.x_dim,
y_start=0,
y_end=c.x_dim,
name="",
show_distribution=False):
plt.figure(f'{name}_correlations_{spectrum_number}', figsize=(15, 12))
#plt.tight_layout(pad=0.4, w_pad=0.5, h_pad=1.0)
plt.title('correlations')
hist_i = hists(post[spectrum_number])
#for i in range(n_parameters-1,-1,-1):
# plt.subplot(n_parameters+1, n_parameters, i+1)
# plt.hist(orig_parameters[:, i+offset], bins=50, histtype='step')
if show_distribution:
for j in range(x_start, x_end):
#ax=plt.subplot(n_parameters+1, n_parameters, j +1)
ax = plt.subplot(y_end - y_start + 1, x_end - x_start,
j - x_start + 1)
plt.step(*(prior_hists[j]), where='post', color='grey')
plt.step(*(hist_i[j]), where='post', color='blue')
#x_low, x_high = np.percentile(post[i][:,j+offset], [q_low, q_high])
plt.plot([x_gt[spectrum_number, j], x_gt[spectrum_number, j]],
[0, 1],
color='red')
#plt.plot([x_low, x_low], [0,1], color='orange')
#plt.plot([x_high, x_high], [0,1], color='orange')
plt.xlabel(f"{c.param_names[j]}")
ax.xaxis.tick_top()
#if j>0:
#ax.axis('off')
# ax.set_yticks([])
#if j==c.co2_pos:
# ax.set_xlim(375,390)
if j > x_start:
ax.set_yticks([])
else:
plt.ylabel(f"Prior dist", fontsize=20)
#for i in range(n_parameters-1,-1,-1):
#for i in range(n_parameters):
# for j in range(n_parameters):
offset = -y_start
if show_distribution:
offset = 1 - y_start
for j in range(y_start, y_end):
for i in range(x_start, x_end):
#print(i,j)
#ax=plt.subplot(n_parameters+1, n_parameters, i+1+n_parameters*(j+1))
ax = plt.subplot(
y_end + offset, x_end - x_start,
i - x_start + 1 + (x_end - x_start) * (j + offset))
plt.subplots_adjust(hspace=.001, wspace=.001)
plt.scatter(x=post[spectrum_number, :, i],
y=post[spectrum_number, :, j],
c="blue",
s=0.5,
alpha=0.1)
plt.scatter(x=x_gt[spectrum_number, i],
y=x_gt[spectrum_number, j],
c='red',
marker="+",
s=150)
if j < y_end - 1:
#ax.axis('off')
ax.set_xticks([])
else:
plt.xlabel(c.param_names[i])
if i > x_start:
ax.set_yticks([])
else:
if len(c.param_names[j]) > 5:
plt.ylabel(
f"{c.param_names[j][:3]}..{c.param_names[j][-3:]}")
else:
plt.ylabel(f"{c.param_names[j]}")
for item in ([ax.title, ax.xaxis.label, ax.yaxis.label]):
#+ ax.get_xticklabels() + ax.get_yticklabels()):
item.set_fontsize(20)
def _plot_multi_pr_curve(self, epoch, type):
"""
:return: avg PR of all folds, list with K PR results
"""
logging.info('***************************** multi pr plot for epoch *************************************')
logging.info('current epoch: ' + str(epoch))
import matplotlib.pyplot as plt
plt.figure()
lw = 2
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.01])
plt.xlabel('Recall')
plt.ylabel('Precision')
num_pr = 0
total_pr = 0.0
best_pr_list = list()
if type == 'max':
for fold_num, pr_epoch_dict in self.pr_max_result_ap_epoch_dict.iteritems(): # fold -> auc, epoch
max_epoch_dict = self.ap_result_dict_all_folds[fold_num][pr_epoch_dict['epoch']]
num_pr += 1
total_pr += max_epoch_dict['ap']
best_pr_list.append(max_epoch_dict['ap'])
plt.step(max_epoch_dict['recall'],
max_epoch_dict['precision'],
color=self.plt_list_colors[fold_num],
alpha=0.6,
where='post',
label='Fold: ' + str(fold_num) + ', epoch:' + str(pr_epoch_dict['epoch']) + ' - (AP = %0.3f)' %
pr_epoch_dict['ap']
)
# plt.fill_between(max_epoch_dict['recall'], max_epoch_dict['precision'], step='post', alpha=0.2, color='b')
else:
for fold_num, epoch_dict in self.ap_result_dict_all_folds.iteritems():
if epoch in epoch_dict:
num_pr += 1
total_pr += epoch_dict[epoch]['ap']
plt.step(epoch_dict[epoch]['recall'],
epoch_dict[epoch]['precision'],
color=self.plt_list_colors[fold_num],
alpha=0.6,
where='post',
label='Fold: ' + str(fold_num) + ' - (AP = %0.3f)' % epoch_dict[epoch]['ap'])
# plt.fill_between(epoch_dict[epoch]['recall'], epoch_dict[epoch]['precision'], step='post', alpha=0.2, color='b')
if num_pr > 0:
mean_ap = float(total_pr) / float(num_pr)
else: # epoch without any result in all folds (due to early stopping)
plt.close()
return
plt.title('AP - epoch number: ' + str(epoch) + ', ' + str(round(mean_ap, 3)))
plt.legend(loc="lower right")
file_suffix = self._get_file_suffix()
import os
plot_dir = '../results/PR/' + \
str(self.vertical_type) + '_' + str(self.df_configuration_dict['y_positive_name']) + '/' \
+ str(file_suffix) + '/' + 'ap_cv' + '/'
if not os.path.exists(plot_dir):
os.makedirs(plot_dir)
plot_path = plot_dir \
+ str(round(mean_ap, 3)) + \
'_epoch=' + str(epoch)
plt.savefig(plot_path + '.png')
plt.close()
logging.info('save PR plot: ' + str(plot_path))
return mean_ap, best_pr_list
sns.heatmap(df_new.corr(),linewidths=0.25,vmax=1.0,annot=True)
cov_mat = np.cov(df_new.iloc[:,:-1].T)
values, vectors = np.linalg.eig(cov_mat)
# Calculation of Explained Variance from the eigenvalues
tot = sum(values)
var_exp = [(i/tot)*100 for i in sorted(values, reverse=True)] # Individual explained variance
cum_var_exp = np.cumsum(var_exp) # Cumulative explained variance
print(list(zip(range(29),cum_var_exp)))
plt.figure(figsize=(10, 10))
plt.bar(range(len(var_exp)), var_exp, alpha=0.3333, label='individual explained variance')
plt.step(range(len(var_exp)), cum_var_exp, where='mid',label='cumulative explained variance')
plt.ylabel('Explained variance ratio')
plt.xlabel('Principal components')
plt.legend(loc='best')
plt.show()
pca = PCA(n_components=17)
X_train, X_test, y_train, y_test = train_test_split(df_new.iloc[:,:-1], df_new.iloc[:,-1], random_state=0)
X_train = pca.fit_transform(X_train)
X_train = pd.DataFrame(X_train)
x_test = pca.transform(X_test)
Model_train = LogisticRegression().fit(X_train, y_train)
def evaluate_and_save_results(best_iteration, theta, data, bucket_indices,
scores, predicted, gold,
shortest_responses_labels, output_file,
image_title, image_file):
acc, cm, roc_auc, pr_auc, ap, f1_max, p_max, r_max, precision, recall, thresholds, MRR, precision_at_1, counter_all_pos, classification_report, classification_report_str = evaluate(
scores, predicted, gold, bucket_indices)
plt.clf()
step_kwargs = ({
'step': 'post'
} if 'step' in signature(plt.fill_between).parameters else {})
plt.step(recall, precision, color='b', alpha=0.2, where='post')
plt.fill_between(recall, precision, alpha=0.2, color='b', **step_kwargs)
plt.xlabel('Recall')
plt.ylabel('Precision')
plt.ylim([0.0, 1.05])
plt.xlim([0.0, 1.0])
print("IMAGE TITLE:", image_title)
plt.title(image_title.format(ap))
plt.savefig(image_file, dpi=300)
with open(output_file, "w") as writer:
writer.write("Best iteration:{}\n".format(best_iteration))
print("Best iteration:{}".format(best_iteration))
writer.write("Accuracy:{}\n".format(acc))
print("Accuracy:{}".format(acc))
writer.write("ROC_AUC_SCORE:{}\n".format(roc_auc))
print("ROC_AUC_SCORE:{}".format(roc_auc))
writer.write("PR_AUC_score:{}\n".format(pr_auc))
print("PR_AUC_score:{}".format(pr_auc))
writer.write("Average Precision Score:{}\n".format(ap))
print("Average Precision Score:{}".format(ap))
writer.write("Max F1:{}\n".format(f1_max))
print("Max F1:{}".format(f1_max))
writer.write("Precision for max F1:{}\n".format(p_max))
print("Precision for max F1:{}".format(p_max))
writer.write("Recall for max F1:{}\n".format(r_max))
print("Recall for max F1:{}".format(r_max))
writer.write("MRR:{}\n".format(MRR))
print("MRR:{}".format(MRR))
writer.write("Precision@1:{}\n".format(precision_at_1))
print("Precision@1:{}".format(precision_at_1))
writer.write("All Pos. Counter:\n{}\n".format(counter_all_pos))
print("All Pos. Counter:\n{}".format(counter_all_pos))
writer.write("CM:\n{}\n".format(cm))
print("CM:\n{}".format(cm))
writer.write(
"Classification report:\n{}\n".format(classification_report_str))
print("Classification report:\n{}".format(classification_report_str))
#feature importance
"""
print(theta.flatten().shape)
feature_importance = np.abs(theta.flatten())
sorted_idx = np.argsort(-feature_importance)
writer.write("Important features:\n")
# print(len(theta))
for idx in sorted_idx:
# print(idx)
writer.write("{}\t{}\t{}\t{}\n".format(idx, feature_names[idx], feature_importance[idx], theta[idx]))
"""
# For each bucket we will print top 10 predictions
writer.write("Predictions:\n")
first_index = 0
counter = 0
for last_index in bucket_indices:
tuples = zip(list(range(first_index,
last_index)), gold[first_index:last_index],
scores[first_index:last_index],
shortest_responses_labels[first_index:last_index])
sorted_by_score = sorted(tuples,
key=lambda tup: tup[2],
reverse=True)
count = 0
# write the shortest response first
qa_set = set()
for i in range(first_index, last_index):
qa_set.add((data.iloc[i][0], data.iloc[i][1]))
if len(qa_set) != 1:
print("QA set:", qa_set)
print("QA set size:", len(qa_set))
print(first_index, last_index)
exit()
l = [(index, gold_label, score, shortest_response_label) for index,
gold_label, score, shortest_response_label in sorted_by_score
if shortest_response_label == 1]
if len(l) != 1:
print("ERROR")
print(l)
for index, gold_label, score, shortest_response_label in l:
print(data.iloc[index][0], data.iloc[index][2])
print("\n")
exit()
index, gold_label, score, shortest_response_label = l[0]
writer.write("Shortest response:{} -- {}\n".format(
data.iloc[index][0], data.iloc[index][2]))
for index, gold_label, score, shortest_response_label in sorted_by_score:
writer.write("{}\t\t{}\t\t{}\t\t{}\t{}\t{}\n".format(
data.iloc[index][0], data.iloc[index][1],
data.iloc[index][2], scores[index], predicted[index],
gold[index]))
assert (gold_label == gold[index])
count += 1
counter += 1
if count == 10:
break
first_index = last_index
if counter >= 5000:
break
return acc, cm, roc_auc, pr_auc, ap, f1_max, p_max, r_max, precision, recall, thresholds, MRR, precision_at_1, counter_all_pos, classification_report, classification_report_str
print('Compute {} feedback...'.format(ctrl_key))
if ctrl_key[-6:] == 'ACADOS':
u[ctrl_key].append(ctrls[ctrl_key[:-7]].step_acados(x[ctrl_key]))
else:
u[ctrl_key].append(ctrls[ctrl_key].step(x[ctrl_key]))
l[ctrl_key].append(tuner.l(x[ctrl_key], u[ctrl_key][-1]) - lOpt)
# forward sim
x[ctrl_key] = plant_sim(x0=x[ctrl_key], p=u[ctrl_key][-1])['xf']
# plot feedback controls to check equivalence
for ctrl_key in ctrl_list:
for i in range(nu):
plt.figure(i)
plt.step(tgrid, [
u[ctrl_key][j][i] - wsol['u', j % N][i] for j in range(len(tgrid))
])
plt.autoscale(enable=True, axis='x', tight=True)
plt.grid(True)
legend = ctrl_list
plt.legend(legend)
plt.title('Feedback control deviation')
plt.figure(nu)
for ctrl_key in ctrl_list:
plt.step(tgrid, l[ctrl_key])
plt.legend(legend)
plt.autoscale(enable=True, axis='x', tight=True)
plt.grid(True)
plt.title('Stage cost deviation')
def show_posterior_histograms(params,
posteriors,
n_plots=n_plots,
orig_prior_hists=hists(
prepare_data.x_to_params(x_all))):
confidence = 0.68
q_low = 100. * 0.5 * (1 - confidence)
q_high = 100. * 0.5 * (1 + confidence)
for i in range(n_plots):
hist_i = hists(posteriors[i])
show_orig_number = n_plots - 10
if i < show_orig_number:
plt.figure(f"orig_{i}", figsize=(15, 7))
for j in range(n_x):
ax = plt.subplot((n_x + 2) / 3, 3, j + 1)
plt.step(*(orig_prior_hists[j]),
where='post',
color='grey',
label="gt dist. of other samples")
plt.step(*(hist_i[j]),
where='post',
color='blue',
label="sampled predictions")
x_low, x_high = np.percentile(posteriors[i][:, j],
[q_low, q_high])
if j == c.co2_pos:
raw_index = prepare_data.ana_names.index("xco2_raw")
unc_index = prepare_data.ana_names.index(
"xco2_uncertainty")
ap_index = prepare_data.ana_names.index("xco2_apriori")
if x_low < 100:
raw_xco2 = ana_all[i, raw_index] - ana_all[i, ap_index]
else:
raw_xco2 = ana_all[i, raw_index]
#x_low=params[i,j]-ana_all[i,unc_index]
#x_high=params[i,j]+ana_all[i,unc_index]
ret_uncert = ana_all[i, unc_index]
ret_bins = np.linspace(params[i, j] - 3 * ret_uncert,
params[i, j] + 3 * ret_uncert, 100)
y_best = scipy.stats.norm.pdf(ret_bins, params[i, j],
ret_uncert)
#scipy.stats.norm.pdf()
plt.plot(ret_bins,
y_best / y_best.max(),
color='brown',
label="uncertainty of gt")
plt.plot([x_low, x_low], [0, 1], color='green')
plt.plot([x_high, x_high], [0, 1],
color='green',
label="1 sigma range")
plt.plot([params[i, j], params[i, j]], [0, 1],
color='red',
linewidth=2,
label="ground truth")
plt.plot([], [], color='brown', label="uncertainty of gt")
unit = ""
if c.param_names[j] == "xco2":
unit = " in ppm"
if c.param_names[j] == "tcwv":
unit = r" in kg $m^{-2}$"
#plt.xlabel(rf"{c.param_names[j]}{unit}")
ax.set_title(rf"{c.param_names[j]}{unit}", fontsize=17)
plt.legend(fontsize=13)
plt.tight_layout()
def monthly_clim(obs_var,
station,
config_file,
logfile="",
plots=False,
diagnostics=False,
winsorize=True):
"""
Run through the variables and pass to the Distributional Gap Checks
:param MetVar obs_var: meteorological variable object
:param Station station: station object
:param str configfile: string for configuration file
:param str logfile: string for log file
:param bool plots: turn on plots
:param bool diagnostics: turn on diagnostic output
"""
flags = np.array(["" for i in range(obs_var.data.shape[0])])
for month in range(1, 13):
month_locs, = np.where(station.months == month)
# note these are for the whole record, just this month is unmasked
normalised_anomalies = prepare_data(obs_var,
station,
month,
diagnostics=diagnostics,
winsorize=winsorize)
if len(normalised_anomalies.compressed()
) >= utils.DATA_COUNT_THRESHOLD:
bins = utils.create_bins(normalised_anomalies, BIN_WIDTH,
obs_var.name)
hist, bin_edges = np.histogram(normalised_anomalies.compressed(),
bins)
try:
upper_threshold = float(
utils.read_qc_config(
config_file, "CLIMATOLOGICAL-{}".format(obs_var.name),
"{}-uthresh".format(month)))
lower_threshold = float(
utils.read_qc_config(
config_file, "CLIMATOLOGICAL-{}".format(obs_var.name),
"{}-lthresh".format(month)))
except KeyError:
print("Information missing in config file")
find_month_thresholds(obs_var,
station,
config_file,
plots=plots,
diagnostics=diagnostics)
upper_threshold = float(
utils.read_qc_config(
config_file, "CLIMATOLOGICAL-{}".format(obs_var.name),
"{}-uthresh".format(month)))
lower_threshold = float(
utils.read_qc_config(
config_file, "CLIMATOLOGICAL-{}".format(obs_var.name),
"{}-lthresh".format(month)))
# now to find the gaps
uppercount = len(
np.where(normalised_anomalies > upper_threshold)[0])
lowercount = len(
np.where(normalised_anomalies < lower_threshold)[0])
if uppercount > 0:
gap_start = utils.find_gap(hist, bins, upper_threshold,
GAP_SIZE)
if gap_start != 0:
bad_locs, = np.ma.where(
normalised_anomalies >
gap_start) # all years for one month
# normalised_anomalies are for the whole record, just this month is unmasked
flags[bad_locs] = "C"
if lowercount > 0:
gap_start = utils.find_gap(hist,
bins,
lower_threshold,
GAP_SIZE,
upwards=False)
if gap_start != 0:
bad_locs, = np.ma.where(
normalised_anomalies <
gap_start) # all years for one month
flags[bad_locs] = "C"
# diagnostic plots
if plots:
import matplotlib.pyplot as plt
plt.clf()
plt.step(bins[1:], hist, color='k', where="pre")
plt.yscale("log")
plt.ylabel("Number of Observations")
plt.xlabel("Scaled {}".format(obs_var.name.capitalize()))
plt.title("{} - month {}".format(station.id, month))
plt.ylim([0.1, max(hist) * 2])
plt.axvline(upper_threshold, c="r")
plt.axvline(lower_threshold, c="r")
bad_locs, = np.where(flags[month_locs] == "C")
bad_hist, dummy = np.histogram(
normalised_anomalies[month_locs][bad_locs], bins)
plt.step(bins[1:], bad_hist, color='r', where="pre")
plt.show()
# append flags to object
obs_var.flags = utils.insert_flags(obs_var.flags, flags)
if diagnostics:
print("Climatological {}".format(obs_var.name))
print(" Cumulative number of flags set: {}".format(
len(np.where(flags != "")[0])))
return # monthly_clim
# Solve the NLP
sol = solver(x0=w0, lbx=lbw, ubx=ubw, lbg=lbg, ubg=ubg, p=0)
# Extract trajectories
x_opt, u_opt = trajectories(sol['x'])
x_opt = x_opt.full() # to numpy array
u_opt = u_opt.full() # to numpy array
# Plot the result
tgrid = np.linspace(0, T, N+1)
plt.figure(1)
plt.clf()
plt.plot(tgrid, x_opt[0], '--')
plt.plot(tgrid, x_opt[1], '-')
plt.step(tgrid, np.append(np.nan, u_opt[0]), '-.')
plt.xlabel('t')
plt.legend(['x1','x2','u'])
plt.grid()
plt.show()
# High-level approach:
# Use factory to e.g. to calculate Hessian of optimal f w.r.t. p
hsolver = solver.factory('h', solver.name_in(), ['sym:hess:f:p:p'])
print('hsolver generated')
hsol = hsolver(x0=sol['x'], lam_x0=sol['lam_x'], lam_g0=sol['lam_g'],
lbx=lbw, ubx=ubw, lbg=lbg, ubg=ubg, p=0)
print('Hessian of f w.r.t. p:')
print(hsol['sym_hess_f_p_p'])
# Low-level approach: Calculate directional derivatives.
# $$ \vdots $$
# $$ y_t - y_0 = a_t \Delta u_0 $$
#
# This is essentially the definition of the step response convolution model described before. What is interesting about this is that we can calculate what y will be at each interval for successive moves in u.
#
# For example purposes only, let us examine a series of three moves "into the future": one now, one at the next time interval, and one two time intervals into the future:
#
# $$ \Delta u_0 = u_0 - u_{-1} $$
# $$ \Delta u_1 = u_1 - u_{0} $$
# $$ \Delta u_2 = u_2 - u_{1} $$
U = np.atleast_2d(np.array([0, 1, 0, 0, -1])).T
print(U)
I_obs = np.pad(np.cumsum(U), (0, 8), 'constant')
plt.step(I_obs, '-bx', where="post")
# then
#
# $$
# \begin{align}
# y_1 - y_0 & = a_1 \Delta u_0 \\
# y_2 - y_0 & = a_2 \Delta u_0 + a_1 \Delta u_1 \\
# y_3 - y_0 & = a_3 \Delta u_0 + a_2 \Delta u_1 + + a_1 \Delta u_2 \\
# y_4 - y_0 & = a_4 \Delta u_0 + a_3 \Delta u_1 + + a_2 \Delta u_2 \\
# \vdots & \\
# y_t - y_0 & = a_t \Delta u_0 + a_{t-1} \Delta u_1 + + a_{t-2} \Delta u_2 \\
# \end{align}
# $$
#
def loop_D_statistic3(name,
popA_list,
popB_list,
popC_list,
popD_list,
popA_ac,
popB_ac,
popC_ac,
popD_ac,
pos,
block_len_snp,
step_len_snp,
cycle="C",
blen=100,
color=[
"blue", "darkorange", "turquoise", "crimson",
"magenta", "limegreen", "forestgreen", "slategray",
"orchid", "darkblue"
]):
windows_pos = allel.moving_statistic(pos,
statistic=lambda v: v[0],
size=block_len_snp,
step=step_len_snp)
# calculate pvalues and focus in this region: duplicated region proper
is_locus = np.logical_and(pos > loc_start, pos < loc_end) # gene region
is_inv = np.logical_and(pos > inv_start, pos < inv_end) # inversion region
# loop
pdf = PdfPages("%s/%s.Dstat_%s.pdf" % (outdir, outcode, name))
colors = cm.rainbow(np.linspace(0, 1, len(popC_list)))
for dn, popD in enumerate(popD_list):
for bn, popB in enumerate(popB_list):
for an, popA in enumerate(popA_list):
print("(((%s,%s),X),%s) chr" % (popA, popB, popD))
fig = plt.figure(figsize=(10, 2))
# whole chromosome: frame
ax1 = plt.subplot(1, 2, 1)
sns.despine(ax=ax1, offset=10)
ax1.set_title("Chr %s (((%s,%s),X),%s)" %
(chrom, popA, popB, popD))
ax1.set_xlim(0, 50)
ax1.set_ylim(-1, 1)
ax1.set_xlabel("Mb")
ax1.set_ylabel("D")
plt.axhline(0, color='k', linestyle="--", label="")
plt.axvline(loc_start / 1e6,
color='red',
linestyle=":",
label="Rdl")
plt.axvline(loc_end / 1e6,
color='red',
linestyle=":",
label="")
plt.axvline(inv_start / 1e6,
color='orange',
linestyle=":",
label="inversion")
plt.axvline(inv_end / 1e6,
color='orange',
linestyle=":",
label="")
ax2 = plt.subplot(1, 4, 3)
sns.despine(ax=ax2, offset=10)
ax2.set_xlim(loc_start / 1e6 - 1, loc_end / 1e6 + 1)
ax2.set_ylim(-1, 1)
ax2.set_xlabel("Mb")
ax2.set_ylabel("D")
plt.axhline(0, color='k', linestyle="--", label="")
plt.axvline(loc_start / 1e6,
color='red',
linestyle=":",
label="Rdl")
plt.axvline(loc_end / 1e6,
color='red',
linestyle=":",
label="")
plt.axvline(inv_start / 1e6,
color='orange',
linestyle=":",
label="inversion")
plt.axvline(inv_end / 1e6,
color='orange',
linestyle=":",
label="")
for cn, popC in enumerate(popC_list):
if popA != popB:
# block-wise patterson D (normalised)
admix_pd_n_win = allel.moving_patterson_d(
aca=popA_ac[popA][:, 0:2],
acb=popB_ac[popB][:, 0:2],
acc=popC_ac[popC][:, 0:2],
acd=popD_ac[popD][:, 0:2],
size=block_len_snp,
step=step_len_snp)
# whole chromosome: plot
plt.subplot(1, 2, 1)
plt.step(windows_pos / 1e6,
admix_pd_n_win,
color=colors[cn])
# estimated D in locus with pval
admix_pd_av_indup = allel.average_patterson_d(
aca=popA_ac[popA][:, 0:2][is_locus],
acb=popB_ac[popB][:, 0:2][is_locus],
acc=popC_ac[popC][:, 0:2][is_locus],
acd=popD_ac[popD][:, 0:2][is_locus],
blen=blen)
# convert Z-score (num of SD from 0) to pval (two-sided)
admix_pd_av_indup_pval = scipy.stats.norm.sf(
abs(admix_pd_av_indup[2])) * 2
# zoomed region: plot
plt.subplot(1, 4, 3)
plt.step(
windows_pos / 1e6,
admix_pd_n_win,
color=colors[cn],
where="post",
label="%s\nD = %.3f +/- %.3f | Z = %.3f | p = %.3E"
%
(popC, admix_pd_av_indup[0], admix_pd_av_indup[1],
admix_pd_av_indup[2], admix_pd_av_indup_pval))
ax2.legend(loc='center left', bbox_to_anchor=(1.1, 0.5))
# save pdf
pdf.savefig(fig, bbox_inches='tight')
pdf.close()
'regular noise 8bits': 'none'
}
grayscale = {
'lower noise 4bits': 'lightgray',
'lower noise 8bits': "lightgray",
'regular noise 4bits': 'lightgray',
'regular noise 8bits': 'lightgray'
}
fig, ax = plt.subplots(tight_layout=True, figsize=(10, 6))
ibin = np.arange(-0.5, nbins + 0.5) # position along the x-axis
# First, plot truth-level distribution
plt.step(list(ibin), [pdata['mean'][0]] + list(pdata['mean']),
label=labels['pdata'],
color='black',
linestyle='dashed')
sns.boxplot(x='bin',
y='unf',
hue='method',
palette=grayscale,
fliersize=0,
data=df,
orient='v')
#stripplot
sns.swarmplot(
x='bin',
y='unf',
hue='method',
def find_month_thresholds(obs_var,
station,
config_file,
plots=False,
diagnostics=False,
winsorize=True):
"""
Use distribution to identify threshold values. Then also store in config file.
:param MetVar obs_var: meteorological variable object
:param Station station: station object
:param str config_file: configuration file to store critical values
:param bool plots: turn on plots
:param bool diagnostics: turn on diagnostic output
:param bool winsorize: apply winsorization at 5%/95%
"""
# get hourly climatology for each month
for month in range(1, 13):
normalised_anomalies = prepare_data(obs_var,
station,
month,
diagnostics=diagnostics,
winsorize=winsorize)
if len(normalised_anomalies.compressed()
) >= utils.DATA_COUNT_THRESHOLD:
bins = utils.create_bins(normalised_anomalies, BIN_WIDTH,
obs_var.name)
hist, bin_edges = np.histogram(normalised_anomalies.compressed(),
bins)
gaussian_fit = utils.fit_gaussian(
bins[1:],
hist,
max(hist),
mu=bins[np.argmax(hist)],
sig=utils.spread(normalised_anomalies))
fitted_curve = utils.gaussian(bins[1:], gaussian_fit)
# diagnostic plots
if plots:
import matplotlib.pyplot as plt
plt.clf()
plt.step(bins[1:], hist, color='k', where="pre")
plt.yscale("log")
plt.ylabel("Number of Observations")
plt.xlabel("Scaled {}".format(obs_var.name.capitalize()))
plt.title("{} - month {}".format(station.id, month))
plt.plot(bins[1:], fitted_curve)
plt.ylim([0.1, max(hist) * 2])
# use bins and curve to find points where curve is < FREQUENCY_THRESHOLD
try:
lower_threshold = bins[1:][np.where(
np.logical_and(fitted_curve < FREQUENCY_THRESHOLD,
bins[1:] < 0))[0]][-1]
except:
lower_threshold = bins[1]
try:
upper_threshold = bins[1:][np.where(
np.logical_and(fitted_curve < FREQUENCY_THRESHOLD,
bins[1:] > 0))[0]][0]
except:
upper_threshold = bins[-1]
if plots:
plt.axvline(upper_threshold, c="r")
plt.axvline(lower_threshold, c="r")
plt.show()
utils.write_qc_config(config_file,
"CLIMATOLOGICAL-{}".format(obs_var.name),
"{}-uthresh".format(month),
"{}".format(upper_threshold),
diagnostics=diagnostics)
utils.write_qc_config(config_file,
"CLIMATOLOGICAL-{}".format(obs_var.name),
"{}-lthresh".format(month),
"{}".format(lower_threshold),
diagnostics=diagnostics)
return # find_month_thresholds
