def plot_commit_differences(project, log=False):
# retrieve all commit sizes for xz
conn = sql.connect('../database.db')
c = conn.cursor()
command = Template("SELECT * FROM Revision WHERE project = '$project'")
command = command.safe_substitute(project=project)
c.execute(command)
xs = []
ys = []
for row in c:
xs.append(row[2])
ys.append(row[4] + row[5])
conn.commit()
zs = [] # time until next commit
for i in xrange(1, len(xs)):
zs.append(xs[i - 1] - xs[i])
#zs = sorted(zs, key=lambda x: x[1], reverse = True)
plt.hist(zs, alpha=0.75)
plt.xlabel('time to next commit [ms]')
plt.ylabel('frequency')
plt.title("Histogram of time between two commits for '" + project + "'")
if log:
plt.yscale('log', nonposy='clip')
#plt.axis([40, 160, 0, 0.03])
plt.grid(True)
plt.show()
def set_xspec(obsid, model = None,
rmffile='/Users/corcoran/Dropbox/nicer_cal/nicer_v0.06.rmf',
arffile='/Users/corcoran/Dropbox/nicer_cal/ni_xrcall_onaxis_v0.06.arf',
workdir='/Users/corcoran/research/WR140/NICER/work/',
ignore='0.0-0.45, 5.-**', showplot=False, showmodel=False
):
"""
sets the pha and model instances for a given obsid
:param obsid: NICER observation id (integer)
:param rmffile: NICER response file
:param arffile: NICER effective area file
:param workdir: NICER dataset working directory
:param ignore: energy range in keV to ignore when fitting
:param showplot: if True plot the model and spectrum vs. energy
:param showmodel: if True print the model parameters
:return:
"""
import xspec
import matplotlib as plt
obs = str(obsid)
xspec.AllData.clear()
xspec.AllModels.clear()
xspecdir = os.path.join(workdir, obs)
phaname = os.path.join(xspecdir, 'ni{obs}_0mpu7_cl.pha'.format(obs=obsid))
print "phaname = {phaname}".format(phaname=phaname)
try:
pha = xspec.Spectrum(phaname)
except:
print "Can't find {phaname}; returning".format(phaname=phaname)
return 0, 0
pha.response = rmffile
pha.response.arf = arffile
pha.ignore(ignore)
#
# Define a model
#
if not model:
model = set_model()
if showmodel:
with sys_pipes():
model.show()
if showplot:
xspec.Plot.setRebin(minSig=3, maxBins=30)
xspec.Plot.device = "/null"
xspec.Plot.xAxis = "keV"
xspec.Plot.yLog = "True"
xspec.Plot("data")
plt.figure(figsize=[10, 6])
plt.yscale('log')
plt.xscale('log')
plt.xlabel('Energy (keV)', fontsize=16)
plt.step(xspec.Plot.x(), xspec.Plot.y())
plt.step(xspec.Plot.x(), xspec.Plot.model())
return pha, model
def animate_plotting(subdir_path,):
average_filename = 'averaged_out.txt'
if os.path.exists( os.path.join(subdir_path,average_filename) ):
print(subdir_path+average_filename+' already exists please use hotPlot.py')
#import existing data for average at the end
# data_out = numpy.genfromtxt(os.path.join(subdir_path,average_filename))
# averaged_data = numpy.array(data_out[:,1])
# angles = data_out[:,0]
#os.remove( os.path.join(subdir_path,average_filename))
else:
files = os.listdir(subdir_path)
#files = [d for d in os.listdir(subdir_path) if os.path.isdir(os.path.join(subdir_path, d))]
onlyfiles_path = [os.path.join(subdir_path,f) for f in files if os.path.isfile(os.path.join(subdir_path,f))]
onlyfiles_path = natsort.natsorted(onlyfiles_path)
averaged_data = []
angles = []
for f in onlyfiles_path:
data = numpy.genfromtxt(f,delimiter = ',')
#data = pandas.read_csv(f)
averaged_data.append(numpy.mean(data))
angle = os.path.basename(f).split('_')[0]
angles.append(float(angle))
fig = plt.plot(angles, averaged_data,'o')
plt.yscale('log')
plt.xscale('log')
plt.legend(loc='upper right')
plt.title(base_path)
plt.grid(True)
plt.xlabel(r'$\theta$ $[deg.]}$')
#plt.xlabel(r'$\mathrm{xlabel\;with\;\LaTeX\;font}$')
plt.ylabel(r'I($\theta$) $[a.u.]$')
def plotLoss(train, test):
import matplotlib as plt
plt.plot(train)
plt.plot(test)
plt.title('Model loss')
plt.ylabel('Loss')
plt.ylim(10**-1.5, 10**3)
plt.yscale('log')
plt.xlabel('Epoch')
plt.legend(['Train', 'Validation'], loc='upper left')
plt.savefig('data/loss.png') # Change later to Files
plt.show()
def show_graph(lr_lists, epochs, steps, out_name='test'):
import matplotlib.pyplot as plt
plt.clf()
plt.rcParams['figure.figsize'] = [20, 5]
x = list(range(epochs * steps))
plt.plot(x, lr_lists, label="line L")
plt.plot()
plt.ylim(10e-5, 1)
plt.yscale("log")
plt.xlabel("iterations")
plt.ylabel("learning rate")
plt.title("Check Cosine Annealing Learing Rate with {}".format(out_name))
plt.legend()
plt.show()
def plot_commit_sizes(project, log=False, releases=True):
# retrieve all commit sizes for xz
conn = sql.connect('../database.db')
c = conn.cursor()
command = Template("SELECT * FROM Revision WHERE project = '$project'")
command = command.safe_substitute(project=project)
c.execute(command)
xs = []
ys = []
for row in c:
xs.append(row[2])
ys.append(row[4] + row[5])
conn.commit()
conn.close()
#zs = sorted(zs, key=lambda x: x[1], reverse = True)
plt.scatter(xs, ys, marker='.', color='black', alpha=0.75, s=20)
plt.xlabel('commit commit [ms]')
plt.ylabel('lines modified [loc]')
plt.title("Commit sizes / commit time for '" + project + "'")
rxs = []
rys = []
if releases:
conn = sql.connect('../database.db')
c = conn.cursor()
command = Template("SELECT * FROM Release WHERE project = '$project'")
command = command.safe_substitute(project=project)
c.execute(command)
for row in c:
rxs.append(row[2])
rys.append(0)
conn.commit()
conn.close()
plt.scatter(rxs, rys, marker='s', color='green', alpha=0.75, s=30)
if log:
plt.yscale('log', nonposy='clip')
#plt.axis([40, 160, 0, 0.03])
plt.grid(True)
plt.show()
def __save(self,n,plot,sfile):
p.figure(figsize=sfile)
p.xlabel(plot.xlabel)
p.ylabel(plot.ylabel)
p.xscale(plot.xscale)
p.yscale(plot.yscale)
p.grid()
for curve in plot.curves:
if curve[1] == None: p.plot(curve[0],curve[2], label=curve[3])
else: p.plot(curve[0], curve[1], curve[2], label=curve[3])
p.rc('legend', fontsize='small')
p.legend(shadow=0, loc='best')
p.axes().set_aspect(plot.aspect)
if not plot.dir: plot.dir = './plots/'
if not plot.name: plot.name = self.__global_name+'_%0*i'%(2,n)
if not os.path.isdir(plot.dir): os.mkdir(plot.dir)
if plot.pgf: p.savefig(plot.dir+plot.name+'.pgf')
else: p.savefig(plot.dir+plot.name+'.pdf', bbox_inches='tight')
p.close()
def fit_spectrum(obsid, pha, model, writexcm=True,
workdir='/Users/corcoran/research/WR140/NICER/work/',
statMethod='cstat'
):
import xspec
from wurlitzer import sys_pipes
import pylab as plt
from heasarc.utils import xspec_utils as xu
if type(pha) != int:
#
# do fit
#
xspec.Fit.statMethod = statMethod
xspec.Fit.perform()
print("Best Fit Model is\n")
with sys_pipes():
model.show()
pha.notice("0.4-10.0")
xspec.Plot.setRebin(minSig=3, maxBins=30)
xspec.Plot.device = "/null"
xspec.Plot.xAxis = "keV"
xspec.Plot.yLog = "True"
xspec.Plot("data")
plt.figure(figsize=[10, 6])
plt.yscale('log')
plt.xscale('log')
plt.xlabel('Energy (keV)', fontsize=16)
plt.title("{obs}".format(obs=obsid), fontsize=16)
plt.errorbar(xspec.Plot.x(), xspec.Plot.y(),
xerr=xspec.Plot.xErr(), yerr=xspec.Plot.yErr(), fmt='.')
plt.step(xspec.Plot.x(), xspec.Plot.model(), where="mid")
band = "0.5-10 keV"
xspec.AllModels.calcFlux(band.replace(' keV', '').replace('-',' '))
print "Flux is {0:.3e} in the {1} band".format(pha.flux[0], band)
if writexcm:
xcmfile = os.path.join(workdir, str(obsid), 'ni{obsid}_0mpu7_cl'.format(obsid=obsid))
xu.write_xcm(xcmfile, pha, model=model)
else:
print "Can't fit OBSID {obs}".format(obs=obsid)
return pha, model
def animate_plotting(subdir_path, ):
average_filename = 'averaged_out.txt'
if os.path.exists(os.path.join(subdir_path, average_filename)):
print(subdir_path + average_filename +
' already exists please use hotPlot.py')
#import existing data for average at the end
# data_out = numpy.genfromtxt(os.path.join(subdir_path,average_filename))
# averaged_data = numpy.array(data_out[:,1])
# angles = data_out[:,0]
#os.remove( os.path.join(subdir_path,average_filename))
else:
files = os.listdir(subdir_path)
#files = [d for d in os.listdir(subdir_path) if os.path.isdir(os.path.join(subdir_path, d))]
onlyfiles_path = [
os.path.join(subdir_path, f) for f in files
if os.path.isfile(os.path.join(subdir_path, f))
]
onlyfiles_path = natsort.natsorted(onlyfiles_path)
averaged_data = []
angles = []
for f in onlyfiles_path:
data = numpy.genfromtxt(f, delimiter=',')
#data = pandas.read_csv(f)
averaged_data.append(numpy.mean(data))
angle = os.path.basename(f).split('_')[0]
angles.append(float(angle))
fig = plt.plot(angles, averaged_data, 'o')
plt.yscale('log')
plt.xscale('log')
plt.legend(loc='upper right')
plt.title(base_path)
plt.grid(True)
plt.xlabel(r'$\theta$ $[deg.]}$')
#plt.xlabel(r'$\mathrm{xlabel\;with\;\LaTeX\;font}$')
plt.ylabel(r'I($\theta$) $[a.u.]$')
# Each column - graphical representation of the distribution of weight in one age group
# Width - proportional to the estimated density - two vertical PDFs printed back to back
# Box plot
sns.boxplot(x='AGE', y='WTKG3', data=data, whis=10) # Whis = 10, turns of feature don't need
plt.show()
# Each box - IQR
# Middle line - median
# Spine - min / max
# With data skewed towards higher values - sometimes useful to look at on logarithmic scale
# Can use pyplot function yscale
sns.boxplot(x='AGE', y='WTKG3', data=data, whis=10) # Whis = 10, turns of feature don't need
plt.yscale('log')
plt.show()
# Correlation
# Correlation coefficient - -1 to 1 - quantifies strength of a linear relationship
# .corr() - result: correlation matrix
# If correl is non-linear, .corr() will generally underestimate the strength of the relationship
# Generate fake data
xs = np.linspace(-1, 1) #Equally spaced points
ys = xs**2
ys += normal(0, 0.05, len(xs)) # x^2 + random noise
# Correl says nothing about slope
# Correl - can use one to predict the other
# Statistic we care about - the slop of the line
plots.append(a)
plots.append( plt.plot(line_x, line_x*lin_fit[0] + lin_fit[1] , '--b', label='$R^2$ = %.2f'%(corr_coef*corr_coef) )) #we append the plot of the line here
kwarg={'size':6 }
plt.legend(loc='upper right', prop=kwarg)
if log2gene1:
plt.xscale('log', basex=2)
plt.xlabel('log2 expression of %s'%in_gene)
plt.xlim(xmax=plt.xlim()[1]+2^10) #make room for the legend
else:
plt.xlabel('%s'%in_gene)
plt.xlim(xmax=plt.xlim()[1]+1000) #make room for the legend
if log2gene2:
plt.yscale('log', basey=2)
plt.ylabel('log2 expression of %s'%in_gene2)
plt.xlim(xmax=plt.xlim()[1]+2^10) #make room for the legend
else:
plt.xlabel('%s'%in_gene)
plt.xlim(xmax=plt.xlim()[1]+1000) #make room for the legend
if organism == 'mouse':
genename1=in_gene.capitalize()
genename2=in_gene2.capitalize()
else: #it is human
genename1= in_gene.upper()
genename2= in_gene2.upper()
def final_decision_plot(df,
figureName,
z_s=10,
z_b=10,
show=False,
block=False,
trafoD_bins=True,
bin_number=15):
"""Plots histogram decision score output of classifier"""
nJets = df['nJ'].tolist()[1]
if trafoD_bins == True:
bins, arg2, arg3 = trafoD_with_error(df)
print(len(bins))
else:
bins = np.linspace(-1, 1, bin_number + 1)
# Initialise plot stuff
plt.ion()
plt.close("all")
fig = plt.figure(figsize=(8.5, 7))
plot_range = (-1, 1)
plot_data = []
plot_weights = []
plot_colors = []
plt.rc('font', weight='bold')
plt.rc('xtick.major', size=5, pad=7)
plt.rc('xtick', labelsize=10)
plt.rcParams["font.weight"] = "bold"
plt.rcParams["axes.labelweight"] = "bold"
plt.rcParams["mathtext.default"] = "regular"
df = setBinCategory(df, bins)
bins = np.linspace(-1, 1, len(bins))
decision_value_list = df['bin_scaled'].tolist()
post_fit_weight_list = df['post_fit_weight'].tolist()
sample_list = df['sample'].tolist()
# Get list of hists.
for t in class_names_grouped[::-1]:
class_names = class_names_map[t]
class_decision_vals = []
plot_weight_vals = []
for c in class_names:
for x in range(0, len(decision_value_list)):
if sample_list[x] == c:
class_decision_vals.append(decision_value_list[x])
plot_weight_vals.append(post_fit_weight_list[x])
plot_data.append(class_decision_vals)
plot_weights.append(plot_weight_vals)
plot_colors.append(colour_map[t])
# Plot.
if nJets == 2:
multiplier = 20
elif nJets == 3:
multiplier = 100
plt.plot([], [],
color='#FF0000',
label=r'VH $\rightarrow$ Vbb x ' + str(multiplier))
plt.hist(plot_data,
bins=bins,
weights=plot_weights,
range=plot_range,
rwidth=1,
color=plot_colors,
label=legend_names[::-1],
stacked=True,
edgecolor='none')
df_sig = df.loc[df['Class'] == 1]
plt.hist(df_sig['bin_scaled'].tolist(),
bins=bins,
weights=(df_sig['post_fit_weight'] * multiplier).tolist(),
range=plot_range,
rwidth=1,
histtype='step',
linewidth=2,
color='#FF0000',
edgecolor='#FF0000')
x1, x2, y1, y2 = plt.axis()
plt.yscale('log', nonposy='clip')
plt.axis((x1, x2, y1, y2 * 1.2))
axes = plt.gca()
axes.set_ylim([5, 135000])
axes.set_xlim([-1, 1])
x = [-1, -0.8, -0.6, -0.4, -0.2, 0, 0.2, 0.4, 0.6, 0.8, 1]
plt.xticks(x, x, fontweight='normal', fontsize=20)
y = [r"10", r"10$^{2}$", r"10$^{3}$", r"10$^{4}$", r"10$^{5}$"]
yi = [10, 100, 1000, 10000, 100000]
plt.yticks(yi, y, fontweight='normal', fontsize=20)
axes.yaxis.set_ticks_position('both')
axes.yaxis.set_tick_params(which='major',
direction='in',
length=10,
width=1)
axes.yaxis.set_tick_params(which='minor',
direction='in',
length=5,
width=1)
axes.xaxis.set_ticks_position('both')
axes.xaxis.set_tick_params(which='major',
direction='in',
length=10,
width=1)
axes.xaxis.set_tick_params(which='minor',
direction='in',
length=5,
width=1)
axes.xaxis.set_minor_locator(AutoMinorLocator(4))
handles, labels = axes.get_legend_handles_labels()
#Weird hack thing to get legend entries in correct order
handles = handles[::-1]
handles = handles + handles
handles = handles[1:12]
plt.legend(loc='upper right',
ncol=1,
prop={'size': 12},
frameon=False,
handles=handles)
plt.ylabel("Events", fontsize=20, fontweight='normal')
axes.yaxis.set_label_coords(-0.07, 0.93)
plt.xlabel(r"BDT$_{VH}$ output", fontsize=20, fontweight='normal')
axes.xaxis.set_label_coords(0.89, -0.07)
an1 = axes.annotate("ATLAS Internal",
xy=(0.05, 0.91),
xycoords=axes.transAxes,
fontstyle='italic',
fontsize=16)
offset_from = OffsetFrom(an1, (0, -1.4))
an2 = axes.annotate(r'$\sqrt{s}$' + " = 13 TeV , 36.1 fb$^{-1}$",
xy=(0.05, 0.91),
xycoords=axes.transAxes,
textcoords=offset_from,
fontweight='normal',
fontsize=12)
offset_from = OffsetFrom(an2, (0, -1.4))
an3 = axes.annotate("1 lepton, " + str(nJets) + " jets, 2 b-tags",
xy=(0.05, 0.91),
xycoords=axes.transAxes,
textcoords=offset_from,
fontstyle='italic',
fontsize=12)
offset_from = OffsetFrom(an3, (0, -1.6))
an4 = axes.annotate("p$^V_T \geq$ 150 GeV",
xy=(0.05, 0.91),
xycoords=axes.transAxes,
textcoords=offset_from,
fontstyle='italic',
fontsize=12)
fig = plt.gcf()
#fig.set_size_inches(10, 4)
plt.savefig(figureName, bbox_inches='tight',
dpi=300) # should before plt.show method
plt.show(block=block)
return fig, axes
