def test_autofmt_xdate(which):
date = [
'3 Jan 2013', '4 Jan 2013', '5 Jan 2013', '6 Jan 2013', '7 Jan 2013',
'8 Jan 2013', '9 Jan 2013', '10 Jan 2013', '11 Jan 2013',
'12 Jan 2013', '13 Jan 2013', '14 Jan 2013'
]
time = [
'16:44:00', '16:45:00', '16:46:00', '16:47:00', '16:48:00', '16:49:00',
'16:51:00', '16:52:00', '16:53:00', '16:55:00', '16:56:00', '16:57:00'
]
angle = 60
minors = [1, 2, 3, 4, 5, 6, 7]
x = mdates.datestr2num(date)
y = mdates.datestr2num(time)
fig, ax = plt.subplots()
ax.plot(x, y)
ax.yaxis_date()
ax.xaxis_date()
ax.xaxis.set_minor_locator(AutoMinorLocator(2))
with warnings.catch_warnings():
warnings.filterwarnings(
'ignore',
'FixedFormatter should only be used together with FixedLocator')
ax.xaxis.set_minor_formatter(FixedFormatter(minors))
with (pytest.warns(mpl.MatplotlibDeprecationWarning)
if which is None else nullcontext()):
fig.autofmt_xdate(0.2, angle, 'right', which)
if which in ('both', 'major', None):
for label in fig.axes[0].get_xticklabels(False, 'major'):
assert int(label.get_rotation()) == angle
if which in ('both', 'minor'):
for label in fig.axes[0].get_xticklabels(True, 'minor'):
assert int(label.get_rotation()) == angle
def plot_diffs(diffs, letters, plt, inf=1e14): L = len(letters) M = diffs.shape[0]/L dmax = np.max(diffs[diffs < inf/100]) * 1.1 + 1 diffs[diffs > inf/100] = dmax plt.imshow(diffs, cmap='gray') ax = get_axis(plt) major_ticks = np.arange(-1, L)*M+(M-.5) minor_ticks = np.arange(L)*M + M/2.0 - 0.5 for axis in (ax.xaxis, ax.yaxis): axis.set_minor_locator(FixedLocator(minor_ticks)) axis.set_minor_formatter(FixedFormatter(letters)) axis.set_major_formatter(NullFormatter()) axis.set_tick_params(which='major', width=1) axis.set_tick_params(which='minor', width=0) ax.set_xticks(major_ticks) ax.set_yticks(major_ticks)
def visualise_ecg(ecg: np.ndarray,
labels: np.ndarray,
pred_vec: np.ndarray = None,
start_offset: int = 0,
end_offest: int = None,
plot_window: int = 300,
max_plots: int = 2) -> None:
"""
:param ecg: ecg signal
:param labels: array of the same length as ecg, containing labels [0, 1, 2, 3, 4]
:param pred_vec: same sa labels, but it is expected to be the model predictions
:param start_offset: plot from given time-step (array index)
:param end_offest: plot until given time-step (array index)
:param plot_window: width of the plot window (array index units)
:param max_plots: max number of plots to show (in case of very long ecg)
:return:
"""
if end_offest is None:
end_offest = ecg.shape[1]
y_formatter = FixedFormatter(["none", "P wave", "QRS", "T wave", "Extra\nsystole"])
y_locator = FixedLocator([0, 1, 2, 3, 4])
for i, start in enumerate(range(start_offset, end_offest, plot_window)):
fig, ax1 = plt.subplots(figsize=(20, 3))
ax1.set_ylabel('ecg', color='blue')
ax1.plot(ecg[0, i:i + plot_window])
ax2 = ax1.twinx()
if pred_vec is not None:
ax2.plot(pred_vec[start:start + plot_window], '.', color='red')
ax2.plot(labels[0, start:start + plot_window] + 0.1, '.', color='green')
ax2.set_ylim(ymin=-0.1, ymax=4.3)
ax2.yaxis.set_major_formatter(y_formatter)
ax2.yaxis.set_major_locator(y_locator)
if i >= max_plots - 1:
print("max_plots was reached, increase it to see more")
break
plt.show()
def make_plot(fps, sens, fps_bs_itp, sens_bs_mean, sens_bs_lb, sens_bs_up,
FROCProbList, numberOfBootstrapSamples, outputDir, CADSystemName):
"""
Plot FROC graphs in log scale.
"""
fps_itp = np.linspace(FROC_minX, FROC_maxX, num=10001)
sens_itp = np.interp(fps_itp, fps, sens)
# create FROC graphs
if len(FROCProbList):
plt.figure()
ax = plt.gca()
clr = 'b'
plt.plot(fps_itp, sens_itp, color=clr, label="%s" % CADSystemName, lw=2)
if numberOfBootstrapSamples:
plt.plot(fps_bs_itp, sens_bs_mean, color=clr, ls='--')
plt.plot(fps_bs_itp, sens_bs_lb, color=clr, ls=':') # , label = "lb")
plt.plot(fps_bs_itp, sens_bs_up, color=clr, ls=':') # , label = "ub")
ax.fill_between(fps_bs_itp, sens_bs_lb, sens_bs_up, facecolor=clr, alpha=0.05)
xmin = FROC_minX
xmax = FROC_maxX
plt.xlim(xmin, xmax)
plt.ylim(0, 1)
plt.xlabel('Average number of false positives per scan')
plt.ylabel('Sensitivity')
plt.legend(loc='lower right')
plt.title('FROC performance - %s' % (CADSystemName))
plt.xscale('log', basex=2)
ax.xaxis.set_major_formatter(FixedFormatter(cpm_xpoints))
# set your ticks manually
ax.xaxis.set_ticks(cpm_xpoints)
ax.yaxis.set_ticks(np.arange(0, 1.1, 0.1))
plt.grid(b=True, which='both')
plt.tight_layout()
plt.savefig(os.path.join(outputDir, "froc_%s.png" % CADSystemName), bbox_inches=0, dpi=300)
def test_autofmt_xdate(which):
date = [
'3 Jan 2013', '4 Jan 2013', '5 Jan 2013', '6 Jan 2013', '7 Jan 2013',
'8 Jan 2013', '9 Jan 2013', '10 Jan 2013', '11 Jan 2013',
'12 Jan 2013', '13 Jan 2013', '14 Jan 2013'
]
time = [
'16:44:00', '16:45:00', '16:46:00', '16:47:00', '16:48:00', '16:49:00',
'16:51:00', '16:52:00', '16:53:00', '16:55:00', '16:56:00', '16:57:00'
]
angle = 60
minors = [1, 2, 3, 4, 5, 6, 7]
x = mdates.datestr2num(date)
y = mdates.datestr2num(time)
fig, ax = plt.subplots()
ax.plot(x, y)
ax.yaxis_date()
ax.xaxis_date()
ax.xaxis.set_minor_locator(AutoMinorLocator(2))
ax.xaxis.set_minor_formatter(FixedFormatter(minors))
fig.autofmt_xdate(0.2, angle, 'right', which)
if which in ('both', 'major', None):
for label in fig.axes[0].get_xticklabels(False, 'major'):
assert int(label.get_rotation()) == angle
if which in ('both', 'minor'):
for label in fig.axes[0].get_xticklabels(True, 'minor'):
assert int(label.get_rotation()) == angle
def set_default_locators_and_formatters(self, axis):
axis.set_major_locator(
FixedLocator(
np.array([1 - 10**(-k) for k in range(1 + self.nines)])))
axis.set_major_formatter(
FixedFormatter([str(1 - 10**(-k)) for k in range(1 + self.nines)]))
def make_scatter(x,
y,
color,
cmap,
barlabel='none',
title='none',
xlab='none',
ylab='none',
xmax=1.0e-36,
ymax=1.0e-36,
xmin=1.0e-36,
ymin=1.0e-36):
if xmax < 1.0e-35:
xmax = np.nanmax(x)
if ymax < 1.0e-35:
ymax = np.nanmax(y)
if xmin < 1.0e-35:
xmin = np.nanmin(x)
if ymin < 1.0e-35:
ymin = np.nanmin(y)
plot_minmaxvals = [xmin, ymin, xmax, ymax]
fig1 = plt.figure(figsize=[20, 12])
gs = plt.GridSpec(100, 100, bottom=0.18, left=0.18, right=0.88)
ax1 = fig1.add_subplot(gs[:, :90])
axC = fig1.add_subplot(gs[:, 95:])
ax1.set_title(title)
ax1.set_xlabel(xlab)
ax1.set_ylabel(ylab)
ax1.set_xlim((xmin, xmax))
ax1.set_ylim((ymin, ymax))
p1 = ax1.scatter(x=x.flatten(),
y=y.flatten(),
c=color.flatten(),
cmap=cmap,
edgecolors='none')
p2 = ax1.plot([xmin, xmax], [ymin, xmax], "r--")
mincol = np.nanmin(color.flatten())
maxcol = np.nanmax(color.flatten())
n = maxcol - mincol + 1
tick_labels = np.unique(color.flatten())
n = tick_labels.count()
# tick_locs = tick_labels * 0.96 + 0.5
tick_locs = np.linspace(mincol, maxcol, n) * 0.98 + 0.75
cbar = fig1.colorbar(p1, ax=ax1, cax=axC)
cbar.locator = FixedLocator(tick_locs)
cbar.formatstrformatter = FormatStrFormatter('%.f')
cbar.formatter = FixedFormatter(tick_labels)
cbar.update_ticks()
axC.text(2.1, 0.5, barlabel, rotation=90, size=15)
plt.show()
def plot_sections(self,
show_traces=True,
show_data=False,
section_names=None,
show_faults=True,
show_topo=True,
figsize=(12, 12)):
assert self.model.solutions.sections is not None, 'no sections for plotting defined'
if self.model.grid.topography is None:
show_topo = False
if section_names is not None:
if type(section_names) == list:
section_names = np.array(section_names)
else:
section_names = self.model.grid.sections.names
if show_traces:
self.plot_section_traces(show_data=show_data,
section_names=section_names,
contour_lines=False)
shapes = self.model.grid.sections.resolution
fig, axes = plt.subplots(nrows=len(section_names),
ncols=1,
figsize=figsize)
for i, section in enumerate(section_names):
j = np.where(self.model.grid.sections.names == section)[0][0]
l0, l1 = self.model.grid.sections.get_section_args(section)
if len(section_names) == 1:
if show_faults:
self.extract_section_fault_lines(section, axes)
if show_topo:
xy = self.make_topography_overlay_4_sections(j)
axes.fill(xy[:, 0], xy[:, 1], 'k', zorder=10)
axes.imshow(self.model.solutions.sections[0][l0:l1].reshape(
shapes[j][0], shapes[j][1]).T,
origin='lower',
cmap=self._cmap,
norm=self._norm,
extent=[
0, self.model.grid.sections.dist[j],
self.model.grid.regular_grid.extent[4],
self.model.grid.regular_grid.extent[5]
])
labels, axname = self._make_section_xylabels(
section,
len(axes.get_xticklabels()) - 2)
pos_list = np.linspace(0, self.model.grid.sections.dist[j],
len(labels))
axes.xaxis.set_major_locator(
FixedLocator(nbins=len(labels), locs=pos_list))
axes.xaxis.set_major_formatter(FixedFormatter((labels)))
axes.set(title=self.model.grid.sections.names[j],
xlabel=axname,
ylabel='Z')
else:
if show_faults:
self.extract_section_fault_lines(section, axes[i])
if show_topo:
xy = self.make_topography_overlay_4_sections(j)
axes[i].fill(xy[:, 0], xy[:, 1], 'k', zorder=10)
axes[i].imshow(self.model.solutions.sections[0][l0:l1].reshape(
shapes[j][0], shapes[j][1]).T,
origin='lower',
cmap=self._cmap,
norm=self._norm,
extent=[
0, self.model.grid.sections.dist[j],
self.model.grid.regular_grid.extent[4],
self.model.grid.regular_grid.extent[5]
])
labels, axname = self._make_section_xylabels(
section,
len(axes[i].get_xticklabels()) - 2)
pos_list = np.linspace(0, self.model.grid.sections.dist[j],
len(labels))
axes[i].xaxis.set_major_locator(
FixedLocator(nbins=len(labels), locs=pos_list))
axes[i].xaxis.set_major_formatter(FixedFormatter((labels)))
axes[i].set(title=self.model.grid.sections.names[j],
xlabel=axname,
ylabel='Z')
fig.tight_layout()
def plotPackedBeam(coordinates,
angle,
axis1,
axis2,
boresight,
equaltorial_range,
pixel_range,
beamRadius,
fileName='pack.png',
scope=1.,
transparency=1.,
index=False,
HD=True):
#angle = 180 - angle
# print("tiling angle: %.2f" % angle)
# angle = angle - 90
thisDpi = 96
matplotlib.rcParams.update({'font.size': 8})
plt.clf()
if HD == True:
fig = plt.figure(figsize=(1600. / thisDpi, 1200. / thisDpi),
dpi=thisDpi)
else:
fig = plt.figure(figsize=(400. / thisDpi, 300. / thisDpi), dpi=thisDpi)
# plt.axes().set_aspect('equal', 'datalim')
axis = fig.add_subplot(111, aspect='equal')
# center = coordinates[0]
for idx in range(len(coordinates)):
coord = coordinates[idx]
ellipse = Ellipse(xy=coord,
width=2 * axis1,
height=2 * axis2,
angle=angle,
alpha=transparency)
ellipse.fill = False
axis.add_artist(ellipse)
if index == True:
axis.text(coord[0],
coord[1],
idx,
size=6,
ha='center',
va='center')
gridCenter = [0, 0]
circle = Ellipse(xy=gridCenter,
width=2 * beamRadius,
height=2 * beamRadius,
angle=0)
circle.fill = False
axis.add_artist(circle)
margin = beamRadius * 1.3 * scope
axis.set_xlim(gridCenter[0] - 2 * margin, gridCenter[0] + 2 * margin)
axis.set_ylim(gridCenter[1] - 2 * margin, gridCenter[1] + 2 * margin)
beamNum = len(coordinates)
xTicks = FixedLocator([pixel_range[0], 0, pixel_range[1]])
xTicksLabel = FixedFormatter([
"{:.2f}".format(equaltorial_range[0]), "{:.2f}".format(boresight[0]),
"{:.2f}".format(equaltorial_range[1])
])
axis.xaxis.set_major_formatter(xTicksLabel)
axis.xaxis.set_major_locator(xTicks)
axis.set_xlabel("RA")
yTicks = FixedLocator([pixel_range[2], 0, pixel_range[3]])
yTicksLabel = FixedFormatter([
"{:.2f}".format(equaltorial_range[2]), "{:.2f}".format(boresight[1]),
"{:.2f}".format(equaltorial_range[3])
])
axis.yaxis.set_major_formatter(yTicksLabel)
axis.yaxis.set_major_locator(yTicks)
axis.yaxis.set_tick_params(rotation=90)
axis.set_ylabel("DEC")
fig.tight_layout()
plt.savefig(fileName, dpi=thisDpi)
plt.close()
def evaluateCAD(seriesUIDs,
results_filename,
outputDir,
allNodules,
CADSystemName,
maxNumberOfCADMarks=-1,
performBootstrapping=False,
numberOfBootstrapSamples=1000,
confidence=0.95):
'''
function to evaluate a CAD algorithm
@param seriesUIDs: list of the seriesUIDs of the cases to be processed
@param results_filename: file with results
@param outputDir: output directory
@param allNodules: dictionary with all nodule annotations of all cases, keys of the dictionary are the seriesuids
@param CADSystemName: name of the CAD system, to be used in filenames and on FROC curve
'''
nodOutputfile = open(os.path.join(outputDir, 'CADAnalysis.txt'), 'w')
nodOutputfile.write("\n")
nodOutputfile.write((60 * "*") + "\n")
nodOutputfile.write("CAD Analysis: %s\n" % CADSystemName)
nodOutputfile.write((60 * "*") + "\n")
nodOutputfile.write("\n")
results = csvTools.readCSV(results_filename)
allCandsCAD = {}
for seriesuid in seriesUIDs:
# collect candidates from result file
nodules = {}
header = results[0]
i = 0
for result in results[1:]:
nodule_seriesuid = result[header.index(seriesuid_label)]
if seriesuid == nodule_seriesuid:
nodule = getNodule(result, header)
nodule.candidateID = i
nodules[nodule.candidateID] = nodule
i += 1
if (maxNumberOfCADMarks > 0):
# number of CAD marks, only keep must suspicous marks
if len(nodules.keys()) > maxNumberOfCADMarks:
# make a list of all probabilities
probs = []
for keytemp, noduletemp in nodules.items():
probs.append(float(noduletemp.CADprobability))
probs.sort(reverse=True) # sort from large to small
probThreshold = probs[maxNumberOfCADMarks]
nodules2 = {}
nrNodules2 = 0
for keytemp, noduletemp in nodules.items():
if nrNodules2 >= maxNumberOfCADMarks:
break
if float(noduletemp.CADprobability) > probThreshold:
nodules2[keytemp] = noduletemp
nrNodules2 += 1
nodules = nodules2
print('adding candidates: ' + seriesuid)
allCandsCAD[seriesuid] = nodules
# open output files
nodNoCandFile = open(
os.path.join(outputDir,
"nodulesWithoutCandidate_%s.txt" % CADSystemName), 'w')
# --- iterate over all cases (seriesUIDs) and determine how
# often a nodule annotation is not covered by a candidate
# initialize some variables to be used in the loop
candTPs = 0
candFPs = 0
candFNs = 0
candTNs = 0
totalNumberOfCands = 0
totalNumberOfNodules = 0
doubleCandidatesIgnored = 0
irrelevantCandidates = 0
minProbValue = -1000000000.0 # minimum value of a float
FROCGTList = []
FROCProbList = []
FPDivisorList = []
excludeList = []
FROCtoNoduleMap = []
ignoredCADMarksList = []
# -- loop over the cases
for seriesuid in seriesUIDs:
# get the candidates for this case
try:
candidates = allCandsCAD[seriesuid]
except KeyError:
candidates = {}
# add to the total number of candidates
totalNumberOfCands += len(candidates.keys())
# make a copy in which items will be deleted
candidates2 = candidates.copy()
# get the nodule annotations on this case
try:
noduleAnnots = allNodules[seriesuid]
except KeyError:
noduleAnnots = []
# - loop over the nodule annotations
for noduleAnnot in noduleAnnots:
# increment the number of nodules
if noduleAnnot.state == "Included":
totalNumberOfNodules += 1
x = float(noduleAnnot.coordX)
y = float(noduleAnnot.coordY)
z = float(noduleAnnot.coordZ)
# 2. Check if the nodule annotation is covered by a candidate
# A nodule is marked as detected when the center of mass of the candidate is within a distance R of
# the center of the nodule. In order to ensure that the CAD mark is displayed within the nodule on the
# CT scan, we set R to be the radius of the nodule size.
diameter = float(noduleAnnot.diameter_mm)
if diameter < 0.0:
diameter = 10.0
radiusSquared = pow((diameter / 2.0), 2.0)
found = False
noduleMatches = []
for key, candidate in candidates.items():
x2 = float(candidate.coordX)
y2 = float(candidate.coordY)
z2 = float(candidate.coordZ)
dist = math.pow(x - x2, 2.) + math.pow(y - y2, 2.) + math.pow(
z - z2, 2.)
if dist < radiusSquared:
if (noduleAnnot.state == "Included"):
found = True
noduleMatches.append(candidate)
if key not in candidates2.keys():
print(
"This is strange: CAD mark %s detected two nodules! Check for overlapping nodule annotations, SeriesUID: %s, nodule Annot ID: %s"
% (str(candidate.id), seriesuid,
str(noduleAnnot.id)))
else:
del candidates2[key]
elif (noduleAnnot.state == "Excluded"
): # an excluded nodule
if bOtherNodulesAsIrrelevant: # delete marks on excluded nodules so they don't count as false positives
if key in candidates2.keys():
irrelevantCandidates += 1
ignoredCADMarksList.append(
"%s,%s,%s,%s,%s,%s,%.9f" %
(seriesuid, -1, candidate.coordX,
candidate.coordY, candidate.coordZ,
str(candidate.id),
float(candidate.CADprobability)))
del candidates2[key]
if len(noduleMatches) > 1: # double detection
doubleCandidatesIgnored += (len(noduleMatches) - 1)
if noduleAnnot.state == "Included":
# only include it for FROC analysis if it is included
# otherwise, the candidate will not be counted as FP, but ignored in the
# analysis since it has been deleted from the nodules2 vector of candidates
if found == True:
# append the sample with the highest probability for the FROC analysis
maxProb = None
for idx in range(len(noduleMatches)):
candidate = noduleMatches[idx]
if (maxProb is None) or (float(
candidate.CADprobability) > maxProb):
maxProb = float(candidate.CADprobability)
FROCGTList.append(1.0)
FROCProbList.append(float(maxProb))
FPDivisorList.append(seriesuid)
excludeList.append(False)
FROCtoNoduleMap.append(
"%s,%s,%s,%s,%s,%.9f,%s,%.9f" %
(seriesuid, noduleAnnot.id, noduleAnnot.coordX,
noduleAnnot.coordY, noduleAnnot.coordZ,
float(noduleAnnot.diameter_mm), str(
candidate.id), float(candidate.CADprobability)))
candTPs += 1
else:
candFNs += 1
# append a positive sample with the lowest probability, such that this is added in the FROC analysis
FROCGTList.append(1.0)
FROCProbList.append(minProbValue)
FPDivisorList.append(seriesuid)
excludeList.append(True)
FROCtoNoduleMap.append(
"%s,%s,%s,%s,%s,%.9f,%s,%s" %
(seriesuid, noduleAnnot.id, noduleAnnot.coordX,
noduleAnnot.coordY, noduleAnnot.coordZ,
float(noduleAnnot.diameter_mm), int(-1), "NA"))
nodNoCandFile.write(
"%s,%s,%s,%s,%s,%.9f,%s\n" %
(seriesuid, noduleAnnot.id, noduleAnnot.coordX,
noduleAnnot.coordY, noduleAnnot.coordZ,
float(noduleAnnot.diameter_mm), str(-1)))
# add all false positives to the vectors
for key, candidate3 in candidates2.items():
candFPs += 1
FROCGTList.append(0.0)
FROCProbList.append(float(candidate3.CADprobability))
FPDivisorList.append(seriesuid)
excludeList.append(False)
FROCtoNoduleMap.append(
"%s,%s,%s,%s,%s,%s,%.9f" %
(seriesuid, -1, candidate3.coordX, candidate3.coordY,
candidate3.coordZ, str(
candidate3.id), float(candidate3.CADprobability)))
if not (len(FROCGTList) == len(FROCProbList) and len(FROCGTList)
== len(FPDivisorList) and len(FROCGTList) == len(FROCtoNoduleMap)
and len(FROCGTList) == len(excludeList)):
nodOutputfile.write(
"Length of FROC vectors not the same, this should never happen! Aborting..\n"
)
nodOutputfile.write("Candidate detection results:\n")
nodOutputfile.write(" True positives: %d\n" % candTPs)
nodOutputfile.write(" False positives: %d\n" % candFPs)
nodOutputfile.write(" False negatives: %d\n" % candFNs)
nodOutputfile.write(" True negatives: %d\n" % candTNs)
nodOutputfile.write(" Total number of candidates: %d\n" %
totalNumberOfCands)
nodOutputfile.write(" Total number of nodules: %d\n" %
totalNumberOfNodules)
nodOutputfile.write(" Ignored candidates on excluded nodules: %d\n" %
irrelevantCandidates)
nodOutputfile.write(
" Ignored candidates which were double detections on a nodule: %d\n"
% doubleCandidatesIgnored)
if int(totalNumberOfNodules) == 0:
nodOutputfile.write(" Sensitivity: 0.0\n")
else:
nodOutputfile.write(" Sensitivity: %.9f\n" %
(float(candTPs) / float(totalNumberOfNodules)))
nodOutputfile.write(" Average number of candidates per scan: %.9f\n" %
(float(totalNumberOfCands) / float(len(seriesUIDs))))
# compute FROC
fps, sens, thresholds = computeFROC(FROCGTList, FROCProbList,
len(seriesUIDs), excludeList)
if performBootstrapping:
fps_bs_itp, sens_bs_mean, sens_bs_lb, sens_bs_up = computeFROC_bootstrap(
FROCGTList,
FROCProbList,
FPDivisorList,
seriesUIDs,
excludeList,
numberOfBootstrapSamples=numberOfBootstrapSamples,
confidence=confidence)
# Write FROC curve
with open(os.path.join(outputDir, "froc_%s.txt" % CADSystemName),
'w') as f:
for i in range(len(sens)):
f.write("%.9f,%.9f,%.9f\n" % (fps[i], sens[i], thresholds[i]))
# Write FROC vectors to disk as well
with open(
os.path.join(outputDir,
"froc_gt_prob_vectors_%s.csv" % CADSystemName),
'w') as f:
for i in range(len(FROCGTList)):
f.write("%d,%.9f\n" % (FROCGTList[i], FROCProbList[i]))
fps_itp = np.linspace(FROC_minX, FROC_maxX, num=10001)
sens_itp = np.interp(fps_itp, fps, sens)
if performBootstrapping:
# Write mean, lower, and upper bound curves to disk
with open(
os.path.join(outputDir,
"froc_%s_bootstrapping.csv" % CADSystemName),
'w') as f:
f.write(
"FPrate,Sensivity[Mean],Sensivity[Lower bound],Sensivity[Upper bound]\n"
)
for i in range(len(fps_bs_itp)):
f.write("%.9f,%.9f,%.9f,%.9f\n" %
(fps_bs_itp[i], sens_bs_mean[i], sens_bs_lb[i],
sens_bs_up[i]))
else:
fps_bs_itp = None
sens_bs_mean = None
sens_bs_lb = None
sens_bs_up = None
# create FROC graphs
if int(totalNumberOfNodules) > 0:
graphTitle = str("")
fig1 = plt.figure()
ax = plt.gca()
clr = 'b'
plt.plot(fps_itp,
sens_itp,
color=clr,
label="%s" % CADSystemName,
lw=2)
if performBootstrapping:
plt.plot(fps_bs_itp, sens_bs_mean, color=clr, ls='--')
plt.plot(fps_bs_itp, sens_bs_lb, color=clr,
ls=':') # , label = "lb")
plt.plot(fps_bs_itp, sens_bs_up, color=clr,
ls=':') # , label = "ub")
ax.fill_between(fps_bs_itp,
sens_bs_lb,
sens_bs_up,
facecolor=clr,
alpha=0.05)
xmin = FROC_minX
xmax = FROC_maxX
plt.xlim(xmin, xmax)
plt.ylim(0, 1)
plt.xlabel('Average number of false positives per scan')
plt.ylabel('Sensitivity')
plt.legend(loc='lower right')
plt.title('FROC performance - %s' % (CADSystemName))
if bLogPlot:
plt.xscale('log', basex=2)
ax.xaxis.set_major_formatter(
FixedFormatter([0.125, 0.25, 0.5, 1, 2, 4, 8]))
# set your ticks manually
ax.xaxis.set_ticks([0.125, 0.25, 0.5, 1, 2, 4, 8])
ax.yaxis.set_ticks(np.arange(0, 1.1, 0.1))
plt.grid(b=True, which='both')
plt.tight_layout()
plt.savefig(os.path.join(outputDir, "froc_%s.png" % CADSystemName),
bbox_inches=0,
dpi=300)
def find_nearest(array, value):
idx = np.where(
(fps - value) == (fps - value)[(fps - value) >= 0].min())[0][-1]
return idx
thres = [0.125, 0.25, 0.5, 1, 2, 4, 8]
sen = []
print('FROC at points: ', thres)
for th in thres:
print('fps: ', th, ', sensitivity: ', sens[find_nearest(fps, th)])
sen.append(sens[find_nearest(fps, th)])
print('=============================================\naverage FROC: ',
np.mean(sen))
return (fps, sens, thresholds, fps_bs_itp, sens_bs_mean, sens_bs_lb,
sens_bs_up)
--------
>>> import matplotlib.pyplot as plt
>>> values = np.arange(12)
>>> plt.figure()
>>> ax = plt.gca()
>>> ax.plot(values)
>>> ax.yaxis.set_major_formatter(librosa.display.ChromaFormatter())
>>> ax.set_ylabel('Pitch class')
'''
def __call__(self, x, pos=None):
'''Format for chroma positions'''
return core.midi_to_note(int(x), octave=False, cents=False)
# A fixed formatter for tonnetz
TONNETZ_FORMATTER = FixedFormatter(
[r'5$_x$', r'5$_y$', r'm3$_x$', r'm3$_y$', r'M3$_x$', r'M3$_y$'])
def cmap(data,
robust=True,
cmap_seq='magma',
cmap_bool='gray_r',
cmap_div='coolwarm'):
'''Get a default colormap from the given data.
If the data is boolean, use a black and white colormap.
If the data has both positive and negative values,
use a diverging colormap.
Otherwise, use a sequential colormap.
from __future__ import division
from builtins import range
from past.utils import old_div
import numpy as np
import matplotlib.pyplot as plt
from boututils.file_import import file_import
from matplotlib.ticker import FixedFormatter, FormatStrFormatter, AutoLocator, AutoMinorLocator
g = file_import("data/cbm18_dens8.grid_nx68ny64.nc")
majorLocator = AutoLocator()
majorFormatter = FormatStrFormatter('%3.0e')
minorLocator = AutoMinorLocator()
Fm = FixedFormatter([
'0', '$1 \\times 10^4$', '$2 \\times 10^4$', '$3 \\times 10^4$',
'$4 \\times 10^4$'
])
Fm2 = FixedFormatter(
['0', '$2 \\times 10^5$', '$4 \\times 10^5$', '$6 \\times 10^5$'])
bxy = g.get('Bxy')
p = g.get('pressure')
jpar0 = g.get('Jpar0')
psixy = g.get('psixy')
btxy = g.get('Btxy')
shiftangle = g.get('ShiftAngle')
nx = g.get('nx')
ny = g.get('ny')
q = np.zeros((nx, ny))
#color = colors[i]
plt.fill_betweenx(np.arange(pos, pos + len(coeffs)),
0,
coeffs,
facecolor=color,
edgecolor=color,
alpha=0.7)
#cmap = "Pastel2"
#plt.fill_betweenx(np.arange(pos, pos + len(coeffs)), 0, coeffs,
# cmap=cmap, alpha=0.7)
ticks.append(pos + len(coeffs) // 2)
pos += len(coeffs) + padding
plt.gca().yaxis.set_major_locator(FixedLocator(ticks))
plt.gca().yaxis.set_major_formatter(FixedFormatter(range(k)))
if k in (3, 5):
plt.ylabel("Cluster")
if k in (5, 6):
plt.gca().set_xticks([-0.1, 0, 0.2, 0.4, 0.6, 0.8, 1])
plt.xlabel("Silhouette Coefficient")
else:
plt.tick_params(labelbottom=False)
score = silhouette_scores[k - 2]
plt.axvline(x=score, color="red", linestyle="--")
plt.title("$k={}, score={:0.2f}$".format(k, score), fontsize=16)
plt.tight_layout()
plt.savefig("../figures/kmeans_silhouette_diagram.pdf", dpi=300)
def add_section(self,
section_name=None,
cell_number=None,
direction='y',
ax=None,
ax_pos=111,
ve=1.,
**kwargs):
extent_val = kwargs.get('extent', None)
self.update_colot_lot()
if ax is None:
ax = self.fig.add_subplot(ax_pos)
if section_name is not None:
if section_name == 'topography':
ax.set_title('Geological map')
ax.set_xlabel('X')
ax.set_ylabel('Y')
extent_val = self.model._grid.topography.extent
else:
dist = self.model._grid.sections.df.loc[section_name, 'dist']
extent_val = [
0, dist, self.model._grid.regular_grid.extent[4],
self.model._grid.regular_grid.extent[5]
]
labels, axname = self._make_section_xylabels(
section_name,
len(ax.get_xticklabels()) - 2)
pos_list = np.linspace(0, dist, len(labels))
ax.xaxis.set_major_locator(
FixedLocator(nbins=len(labels), locs=pos_list))
ax.xaxis.set_major_formatter(FixedFormatter((labels)))
ax.set(title=section_name, xlabel=axname, ylabel='Z')
elif cell_number is not None:
_a, _b, _c, extent_val, x, y = self._slice(direction,
cell_number)[:-2]
ax.set_xlabel(x)
ax.set_ylabel(y)
ax.set(title='Cell Number: ' + str(cell_number) + ' Direction: ' +
str(direction))
if extent_val is not None:
if extent_val[3] < extent_val[
2]: # correct vertical orientation of plot
ax.invert_yaxis()
self._aspect = (extent_val[3] - extent_val[2]) / (
extent_val[1] - extent_val[0]) / ve
ax.set_xlim(extent_val[0], extent_val[1])
ax.set_ylim(extent_val[2], extent_val[3])
ax.set_aspect('equal')
# Adding some properties to the axes to make easier to plot
ax.section_name = section_name
ax.cell_number = cell_number
ax.direction = direction
ax.tick_params(axis='x', labelrotation=30)
self.axes = np.append(self.axes, ax)
self.fig.tight_layout()
return ax
ax.vlines(x=Vrmax, ymin=0.7, ymax=pr0, linestyles='dashed', lw=1)
# Ejes
ax.set_xlabel(r'$v$')
ax.set_ylabel(r'$p$', rotation='horizontal', labelpad=15)
ax.set_ylim((0.7, 1.0))
# Ticks especificos
ax.xaxis.set_ticks_position('bottom')
x_formatter = FixedFormatter(['$v_l$', '$v_g$'])
x_locator = FixedLocator([Vrmin, Vrmax])
ax.xaxis.set_major_formatter(x_formatter)
ax.xaxis.set_major_locator(x_locator)
y_formatter = FixedFormatter([r'$p_0$'])
y_locator = FixedLocator([pr0])
ax.yaxis.set_major_formatter(y_formatter)
ax.yaxis.set_major_locator(y_locator)
def dinuc_plot(data, graph_name="ADinucTester.png", title = "", \
gene_type_formats=standard_gene_type_formats, \
background_line_color=light_gray, avg_formats={},
point_formats={}):
"""Returns dinucleotide usage plot from given species data.
Data is a dict with the following structure:
{name: {gene_type: [values]}}
where name is e.g. the name of the species (will be displayed on graph),
and gene_type will be the name of the type of gene (e.g. ribosomal,
hgt, non-hgt, all).
Values should be the 16-element array obtained from calling normalize() on
the DinucUsage objects giving the DinucUsage frequencies, as fractions,
for each of the 16 dinucleotides in alphabet order (TT -> GG).
Calculates the average of each gene type on the fly and connects these
averages with lines. If you have precalculated the average, just pass it
in as a single 'gene' (the average of an array with one item will be
itself...).
avg_formats should be passable to plot(), e.g. markersize.
point_formats should be passable to scatter(), e.g. 'c'.
"""
#constructing the vertical lines
for a in line_positions:
axvline(a,ymin=0,ymax=1, color=background_line_color)
curr_color_index = 0
lines = []
labels = []
for label, gene_types in sorted(data.items()):
gene_types = data[label]
curr_color = colors[curr_color_index]
for type_label, usages in sorted(gene_types.items()):
labels.append(':'.join(map(str, [label,type_label])))
x_positions = line_positions + \
offsets.get(type_label, offsets[None])
curr_format = gene_type_formats.get(type_label, \
gene_type_formats[None])
#plot the averages...connected with a line for easier visual
avg_format = curr_format.copy()
avg_format.update(avg_formats)
avg = sum(array(usages))/len(usages)
curr_line = plot(x_positions, avg, color=curr_color,
markerfacecolor=curr_color, label=label,
**avg_format)
lines.append(curr_line)
point_format = curr_format.copy()
if len(usages) > 1: #got more than one gene: plot points
point_format.update(point_formats)
for u in usages:
scatter(x_positions, u, c=curr_color, **point_format)
#get the next color
try:
curr_color_index += 1
except IndexError: #fell off end of list -- wrap around
curr_color_index = 0
legend(lines, labels, loc='best', \
prop=FontProperties(size='smaller'))
a = gca()
a.set_yticks(arange(0,1.01,.1)) #set the yaxis labels
a.set_xticks(arange(0.5,16.5,1))
a.xaxis.set_major_formatter(FixedFormatter(axis_names))
xlim(0,16)
if graph_name is not None:
savefig(graph_name)
def plotConfusion(confusion,
map,
grid_spacing,
title,
file_name,
inset=None,
fudge=16):
pylab.figure()
axmain = pylab.axes()
gs_x, gs_y = grid_spacing
confusion_cmap_dict = {
'red': (
(0.0, 0.5, 0.5),
(0.25, 0.5, 1.0),
(0.5, 1.0, 0.0),
(0.75, 0.0, 1.0),
(1.0, 1.0, 1.0),
),
'green': (
(0.0, 0.5, 0.5),
(0.25, 0.5, 0.0),
(0.5, 0.0, 1.0),
(0.75, 1.0, 1.0),
(1.0, 1.0, 1.0),
),
'blue': (
(0.0, 0.5, 0.5),
(0.25, 0.5, 1.0),
(0.5, 1.0, 0.0),
(0.75, 0.0, 1.0),
(1.0, 1.0, 1.0),
),
}
confusion_cmap = LinearSegmentedColormap('confusion', confusion_cmap_dict,
256)
nx, ny = confusion.shape
xs, ys = np.meshgrid(gs_x * np.arange(nx), gs_y * np.arange(ny))
pylab.pcolormesh(xs, ys, confusion, cmap=confusion_cmap, vmin=0, vmax=3)
tick_locs = [0.375 + 0.75 * l for l in range(4)]
tick_labels = ["Correct\nNegative", "False\nAlarm", "Miss", "Hit"]
bar = pylab.colorbar()
bar.locator = FixedLocator(tick_locs)
bar.formatter = FixedFormatter(tick_labels)
pylab.setp(pylab.getp(bar.ax, 'ymajorticklabels'), fontsize='large')
bar.update_ticks()
drawPolitical(map, scale_len=25)
if inset:
lb_y, lb_x = [b.start for b in inset]
ub_y, ub_x = [b.stop + fudge for b in inset]
inset_exp = (slice(lb_y, ub_y), slice(lb_x, ub_x))
axins = zoomed_inset_axes(pylab.gca(), 2, loc=4)
pylab.sca(axins)
pylab.pcolormesh(xs[inset_exp],
ys[inset_exp],
confusion[inset_exp],
cmap=confusion_cmap,
vmin=0,
vmax=3)
drawPolitical(map)
pylab.xlim([lb_x * gs_x, (ub_x - 1) * gs_x])
pylab.ylim([lb_y * gs_y, (ub_y - 1) * gs_y])
mark_inset(axmain, axins, loc1=1, loc2=3, fc='none', ec='k')
pylab.sca(axmain)
pylab.suptitle(title)
pylab.savefig(file_name)
pylab.close()
return
picture = ax.pcolorfast(
x,
y,
Z,
cmap='jet',
norm=colors.LogNorm(vmin=1e3, vmax=1e6), # logarithmic scaling
)
# x axis:
# after https://stackoverflow.com/questions/17129947/how-to-set-ticks-on-fixed-position-matplotlib
name_list = [elt.strftime('%a %d %b') for elt in x_list]
x_labels_interval = 5 * 24 # each 5 days
ax.xaxis.set_major_locator(FixedLocator(x[::x_labels_interval]))
ax.xaxis.set_major_formatter(
FixedFormatter(name_list[::x_labels_interval]))
plt.xticks(rotation=70)
# y axis:
ax.set_yscale('log')
ax.yaxis.set_major_formatter(FormatStrFormatter('%.1f'))
ax.yaxis.set_minor_formatter(FormatStrFormatter('%.1f'))
plt.ylim(ymin=0.5, ymax=np.max(y))
cb = plt.colorbar(picture)
cb.set_label('Arbitrary Units')
plt.tight_layout()
plt.savefig(f"{results_repository}/{title}.png", format='png')
length = (time.time() - start_time)
def evaluateCAD(seriesUIDs,
results_filename,
outputDir,
allNodules,
CADSystemName,
maxNumberOfCADMarks=-1,
performBootstrapping=False,
numberOfBootstrapSamples=1000,
confidence=0.95):
'''
function to evaluate a CAD algorithm
@param seriesUIDs: 所有的测试集CT图像名称列表
@param results_filename: 提交的csv文件，*.csv
@param outputDir: 存放F-ROC计算结果的文件夹路径
@param allNodules: 所有的nodule构成的字典，以图像名索引，GT
@param CADSystemName: 系统名字，用来作为文件的前缀之类
@param maxNumberOfCADMarks: 一张CT图像最多允许多少条标注
@param performBootstrapping:
@param numberOfBootstrapSamples:
@param confidence:
'''
nodOutputfile = open(os.path.join(outputDir, 'CADAnalysis.txt'), 'w')
nodOutputfile.write("\n")
nodOutputfile.write((60 * "*") + "\n")
nodOutputfile.write("CAD Analysis: %s\n" % CADSystemName)
nodOutputfile.write((60 * "*") + "\n")
nodOutputfile.write("\n")
results = csvTools.readCSV(results_filename) # 最终的csv文件结果
allCandsCAD = {}
for seriesuid in seriesUIDs: # 对每一个测试图像ID
# collect candidates from result file
nodules = {}
header = results[0] # csv文件第一行，表头
i = 0
for result in results[1:]: # 对于每一个标注
nodule_seriesuid = result[header.index(seriesuid_label)] # 该标注的文件名
if seriesuid == nodule_seriesuid: # 判断该标注的是否是suriesuid
nodule = getNodule(result, header)
nodule.candidateID = i
nodules[nodule.candidateID] = nodule # 同一个ID的所有nodule
i += 1
if (maxNumberOfCADMarks > 0):
# 如果一张CT图像的标注超过某个值，就按照得分排序，只截取前maxNumberOfCADMarks条记录
if len(nodules.keys()) > maxNumberOfCADMarks:
# make a list of all probabilities
probs = []
for keytemp, noduletemp in nodules.iteritems():
probs.append(float(noduletemp.CADprobability))
probs.sort(reverse=True) # sort from large to small
probThreshold = probs[maxNumberOfCADMarks]
nodules2 = {}
nrNodules2 = 0
for keytemp, noduletemp in nodules.iteritems():
if nrNodules2 >= maxNumberOfCADMarks:
break
if float(noduletemp.CADprobability) > probThreshold:
nodules2[keytemp] = noduletemp
nrNodules2 += 1
nodules = nodules2
print 'adding candidates: ' + seriesuid
allCandsCAD[seriesuid] = nodules # 以图像名称索引nodule字典
# open output files
nodNoCandFile = open(
os.path.join(outputDir,
"nodulesWithoutCandidate_%s.txt" % CADSystemName), 'w')
# --- iterate over all cases (seriesUIDs) and determine how
# often a nodule annotation is not covered by a candidate
# initialize some variables to be used in the loop
candTPs = 0
candFPs = 0
candFNs = 0
candTNs = 0
totalNumberOfCands = 0
totalNumberOfNodules = 0
doubleCandidatesIgnored = 0
irrelevantCandidates = 0
minProbValue = -1000000000.0 # minimum value of a float
FROCGTList = []
FROCProbList = []
FPDivisorList = []
excludeList = []
FROCtoNoduleMap = []
ignoredCADMarksList = []
# -- loop over the cases
for seriesuid in seriesUIDs: # 对于每一张CT图像
# get the candidates for this case
try:
candidates = allCandsCAD[seriesuid] # 该图像的预测标注信息
except KeyError:
candidates = {}
totalNumberOfCands += len(candidates.keys()) # 预测标注总个数
# make a copy in which items will be deleted
candidates2 = candidates.copy() # 复制该图像的预测标注信息
# get the nodule annotations on this case
try:
noduleAnnots = allNodules[seriesuid] # 该图像的GT标注信息
except KeyError:
noduleAnnots = []
# - loop over the nodule annotations
for noduleAnnot in noduleAnnots: # 对GT标注中的每一条记录
# increment the number of nodules
if noduleAnnot.state == "Included": # 该标注被用来计算结果
totalNumberOfNodules += 1 # 记录GT标注总数
x = float(noduleAnnot.coordX) # GT标注 X坐标
y = float(noduleAnnot.coordY) # GT标注 Y坐标
z = float(noduleAnnot.coordZ) # GT标注 Z坐标
# 2. Check if the nodule annotation is covered by a candidate
# A nodule is marked as detected when the center of mass of the candidate is within a distance R of
# the center of the nodule. In order to ensure that the CAD mark is displayed within the nodule on the
# CT scan, we set R to be the radius of the nodule size.
diameter = float(noduleAnnot.diameter_mm) # GT标注的直径
if diameter < 0.0:
diameter = 5
radiusSquared = pow((diameter / 2.0), 2.0) # GT 半径的平方
found = False
noduleMatches = []
for key, candidate in candidates.iteritems(): # 遍历预测的每一条标注
x2 = float(candidate.coordX) # 预测的坐标 X
y2 = float(candidate.coordY) # 预测的坐标 Y
z2 = float(candidate.coordZ) # 预测的坐标 Z
dist = math.pow(x - x2, 2.) + math.pow(y - y2, 2.) + math.pow(
z - z2, 2.) # 预测与真实的距离的差
if dist < radiusSquared: # 如果距离小于半径
if (noduleAnnot.state == "Included"): # 可被用于测评的标注
found = True
noduleMatches.append(
candidate) # 将该条预测的标注添加到 noduleMatches
if key not in candidates2.keys():
print "This is strange: CAD mark %s detected two nodules! Check for overlapping nodule annotations, SeriesUID: %s, nodule Annot ID: %s" % (
str(candidate.id), seriesuid,
str(noduleAnnot.id))
else:
del candidates2[key] # 在candidates2中将相应数据删除
elif (noduleAnnot.state == "Excluded"
): # an excluded nodule
if bOtherNodulesAsIrrelevant: # delete marks on excluded nodules so they don't count as false positives
if key in candidates2.keys():
irrelevantCandidates += 1
ignoredCADMarksList.append(
"%s,%s,%s,%s,%s,%s,%.9f" %
(seriesuid, -1, candidate.coordX,
candidate.coordY, candidate.coordZ,
str(candidate.id),
float(candidate.CADprobability)))
del candidates2[key]
if len(noduleMatches) > 1: # 如果预测的标注中有至少两个都预测到了GT的某一个nodule
doubleCandidatesIgnored += (len(noduleMatches) - 1
) # 舍弃多余的标注，记录舍弃的数目
if noduleAnnot.state == "Included": # 判断GT中的这条标注可被用来计算F-ROC
# only include it for FROC analysis if it is included
# otherwise, the candidate will not be counted as FP, but ignored in the
# analysis since it has been deleted from the nodules2 vector of candidates
if found == True: # 对该条GT标注，在预测标注中，找到了至少一条符合的
# append the sample with the highest probability for the FROC analysis
maxProb = None
for idx in range(
len(noduleMatches)): # 对所有符合条件的预测，寻找最大的概率的一条
candidate = noduleMatches[idx]
if (maxProb is None) or (float(
candidate.CADprobability) > maxProb):
maxProb = float(candidate.CADprobability)
FROCGTList.append(1.0)
FROCProbList.append(float(maxProb))
FPDivisorList.append(seriesuid)
excludeList.append(False)
FROCtoNoduleMap.append(
"%s,%s,%s,%s,%s,%.9f,%s,%.9f" %
(seriesuid, noduleAnnot.id, noduleAnnot.coordX,
noduleAnnot.coordY, noduleAnnot.coordZ,
float(noduleAnnot.diameter_mm), str(
candidate.id), float(candidate.CADprobability)))
candTPs += 1
else:
candFNs += 1
# append a positive sample with the lowest probability, such that this is added in the FROC analysis
FROCGTList.append(1.0)
FROCProbList.append(minProbValue)
FPDivisorList.append(seriesuid)
excludeList.append(True)
FROCtoNoduleMap.append(
"%s,%s,%s,%s,%s,%.9f,%s,%s" %
(seriesuid, noduleAnnot.id, noduleAnnot.coordX,
noduleAnnot.coordY, noduleAnnot.coordZ,
float(noduleAnnot.diameter_mm), int(-1), "NA"))
nodNoCandFile.write(
"%s,%s,%s,%s,%s,%.9f,%s\n" %
(seriesuid, noduleAnnot.id, noduleAnnot.coordX,
noduleAnnot.coordY, noduleAnnot.coordZ,
float(noduleAnnot.diameter_mm), str(-1)))
# add all false positives to the vectors
for key, candidate3 in candidates2.iteritems():
candFPs += 1
FROCGTList.append(0.0)
FROCProbList.append(float(candidate3.CADprobability))
FPDivisorList.append(seriesuid)
excludeList.append(False)
FROCtoNoduleMap.append(
"%s,%s,%s,%s,%s,%s,%.9f" %
(seriesuid, -1, candidate3.coordX, candidate3.coordY,
candidate3.coordZ, str(
candidate3.id), float(candidate3.CADprobability)))
if not (len(FROCGTList) == len(FROCProbList) and len(FROCGTList)
== len(FPDivisorList) and len(FROCGTList) == len(FROCtoNoduleMap)
and len(FROCGTList) == len(excludeList)):
nodOutputfile.write(
"Length of FROC vectors not the same, this should never happen! Aborting..\n"
)
nodOutputfile.write("Candidate detection results:\n")
nodOutputfile.write(" True positives: %d\n" % candTPs)
nodOutputfile.write(" False positives: %d\n" % candFPs)
nodOutputfile.write(" False negatives: %d\n" % candFNs)
nodOutputfile.write(" True negatives: %d\n" % candTNs)
nodOutputfile.write(" Total number of candidates: %d\n" %
totalNumberOfCands)
nodOutputfile.write(" Total number of nodules: %d\n" %
totalNumberOfNodules)
nodOutputfile.write(" Ignored candidates on excluded nodules: %d\n" %
irrelevantCandidates)
nodOutputfile.write(
" Ignored candidates which were double detections on a nodule: %d\n"
% doubleCandidatesIgnored)
if int(totalNumberOfNodules) == 0:
nodOutputfile.write(" Sensitivity: 0.0\n")
else:
nodOutputfile.write(" Sensitivity: %.9f\n" %
(float(candTPs) / float(totalNumberOfNodules)))
nodOutputfile.write(" Average number of candidates per scan: %.9f\n" %
(float(totalNumberOfCands) / float(len(seriesUIDs))))
# compute FROC
fps, sens, thresholds = computeFROC(FROCGTList, FROCProbList,
len(seriesUIDs), excludeList)
if performBootstrapping:
fps_bs_itp, sens_bs_mean, sens_bs_lb, sens_bs_up = computeFROC_bootstrap(
FROCGTList,
FROCProbList,
FPDivisorList,
seriesUIDs,
excludeList,
numberOfBootstrapSamples=numberOfBootstrapSamples,
confidence=confidence)
# Write FROC curve
with open(os.path.join(outputDir, "froc_%s.txt" % CADSystemName),
'w') as f:
for i in range(len(sens)):
f.write("%.9f,%.9f,%.9f\n" % (fps[i], sens[i], thresholds[i]))
# Write FROC vectors to disk as well
with open(
os.path.join(outputDir,
"froc_gt_prob_vectors_%s.csv" % CADSystemName),
'w') as f:
for i in range(len(FROCGTList)):
f.write("%d,%.9f\n" % (FROCGTList[i], FROCProbList[i]))
fps_itp = np.linspace(FROC_minX, FROC_maxX, num=10001)
sens_itp = np.interp(fps_itp, fps, sens)
sum_sensitivity = 0
for idx in range(len(fps_itp) - 1):
if fps_itp[idx] < 0.125 and fps_itp[idx + 1] > 0.125:
print("0.125:", sens_itp[idx])
sum_sensitivity += sens_itp[idx]
if fps_itp[idx] < 0.25 and fps_itp[idx + 1] > 0.25:
print("0.25:", sens_itp[idx])
sum_sensitivity += sens_itp[idx]
if fps_itp[idx] < 0.5 and fps_itp[idx + 1] > 0.5:
print("0.5:", sens_itp[idx])
sum_sensitivity += sens_itp[idx]
if fps_itp[idx] < 1 and fps_itp[idx + 1] > 1:
print("1:", sens_itp[idx])
sum_sensitivity += sens_itp[idx]
if fps_itp[idx] < 2 and fps_itp[idx + 1] > 2:
print("2:", sens_itp[idx])
sum_sensitivity += sens_itp[idx]
if fps_itp[idx] < 4 and fps_itp[idx + 1] > 4:
print("4:", sens_itp[idx])
sum_sensitivity += sens_itp[idx]
if fps_itp[idx] < 8 and fps_itp[idx + 1] > 8:
print("8:", sens_itp[idx])
sum_sensitivity += sens_itp[idx]
ave_sensitivity = sum_sensitivity / 7.0
print("final score is %d" % ave_sensitivity)
if performBootstrapping:
# Write mean, lower, and upper bound curves to disk
with open(
os.path.join(outputDir,
"froc_%s_bootstrapping.csv" % CADSystemName),
'w') as f:
f.write(
"FPrate,Sensivity[Mean],Sensivity[Lower bound],Sensivity[Upper bound]\n"
)
for i in range(len(fps_bs_itp)):
f.write("%.9f,%.9f,%.9f,%.9f\n" %
(fps_bs_itp[i], sens_bs_mean[i], sens_bs_lb[i],
sens_bs_up[i]))
else:
fps_bs_itp = None
sens_bs_mean = None
sens_bs_lb = None
sens_bs_up = None
# create FROC graphs
if int(totalNumberOfNodules) > 0:
graphTitle = str("")
fig1 = plt.figure()
ax = plt.gca()
clr = 'b'
plt.plot(fps_itp,
sens_itp,
color=clr,
label="%s" % CADSystemName,
lw=2)
if performBootstrapping:
plt.plot(fps_bs_itp, sens_bs_mean, color=clr, ls='--')
plt.plot(fps_bs_itp, sens_bs_lb, color=clr,
ls=':') # , label = "lb")
plt.plot(fps_bs_itp, sens_bs_up, color=clr,
ls=':') # , label = "ub")
ax.fill_between(fps_bs_itp,
sens_bs_lb,
sens_bs_up,
facecolor=clr,
alpha=0.05)
xmin = FROC_minX
xmax = FROC_maxX
plt.xlim(xmin, xmax)
plt.ylim(0, 1)
plt.xlabel('Average number of false positives per scan')
plt.ylabel('Sensitivity')
plt.legend(loc='lower right')
plt.title('FROC performance - %s' % (CADSystemName))
if bLogPlot:
plt.xscale('log', basex=2)
ax.xaxis.set_major_formatter(
FixedFormatter([0.125, 0.25, 0.5, 1, 2, 4, 8]))
# set your ticks manually
ax.xaxis.set_ticks([0.125, 0.25, 0.5, 1, 2, 4, 8])
ax.yaxis.set_ticks(np.arange(0, 1.1, 0.1))
plt.grid(b=True, which='both')
plt.tight_layout()
plt.savefig(os.path.join(outputDir, "froc_%s.png" % CADSystemName),
bbox_inches=0,
dpi=300)
return (fps, sens, thresholds, fps_bs_itp, sens_bs_mean, sens_bs_lb,
sens_bs_up)
partialSum2 = np.sum(omega[:, j, :] * (ratioEa[:, j, :]), axis=1)
mp.plotRandM(co2, partialSum1, partialSum2, axarr[maxRanges - 1 - j],
handles, j == maxRanges - 1)
plt.savefig("multiplicitiesRandM" + ext + ".png", bbox_inches='tight')
lastOmegas = np.zeros(shape=(maxRanges, maxAlfa - minAlfa))
epsilon = np.zeros(shape=(maxCo2, maxRanges, p.maxA - p.minA))
if omegas:
co2.append(maxCo2)
labels = ["0", "20", "40", "60", "80", "100"]
cm = plt.get_cmap('tab20')
for j in range(
0, maxRanges): # different temperature ranges (low, medium, high)
axarr[maxRanges - 1 - j].get_xaxis().set_major_formatter(
FixedFormatter(labels))
partialSum = np.sum(omega[:, j, :] *
(ratioEa[:, j, :] - multiplicityEa[:, j, :]),
axis=1)
lgs = []
for i, a in enumerate(range(minAlfa, maxAlfa)): #alfa
lgs.append(axarr[maxRanges - 1 - j].fill_between(
co2,
partialSum,
color=cm(a / (maxAlfa - 1)),
label=labelAlfa[a]))
lastOmegas[maxRanges - 1 - j, i] = partialSum[-1]
partialSum -= omega[:, j, i] * (ratioEa[:, j, i] -
multiplicityEa[:, j, i])
epsilon[:, j, i] = omega[:, j, i] * (ratioEa[:, j, i] -
multiplicityEa[:, j, i])
def plot_profile(profile,
fname=None,
scale=1,
fillcolors=('r', 'b'),
trim=None,
top=None,
moveout_model='iasp91'):
"""
Plot receiver function profile.
:param profile: stream holding the profile
:param fname: filename to save plot to. Can be None. In this case
the figure is left open.
:param scale: scale for individual traces
:param fillcolors: fill colors for positive and negative wiggles
:param trim: trim stream relative to onset before plotting using
`~.rfstream.RFStream.slice2()`
:param top: show second axes on top of profile with additional information.
Valid values: 'hist' - Plot histogram showing the number of receiver
functions stacked in the corresponding bin
:param moveout_model: string with model filename. Will be loaded into a
`~.simple_model.SimpleModel` object to calculate depths for
tick labels.
"""
if len(profile) == 0:
return
if trim:
profile = profile.slice2(*trim, reftime='onset')
fig = plt.figure()
ax = fig.add_axes([0.1, 0.1, 0.8, 0.7])
widths = [tr.stats.box_length for tr in profile]
pad = max(1, scale) * min(widths)
xlim = (min(tr.stats.box_pos for tr in profile) - pad,
max(tr.stats.box_pos for tr in profile) + pad)
max_ = max(np.max(np.abs(tr.data)) for tr in profile)
for tr in profile:
x = tr.stats.box_pos + scale * tr.data / max_ * min(widths)
y = tr.times() - (tr.stats.onset - tr.stats.starttime)
ax.plot(x, y, 'k')
c1, c2 = fillcolors
if c1:
ax.fill_betweenx(y,
x,
tr.stats.box_pos,
where=x >= tr.stats.box_pos,
facecolor=c1)
if c2:
ax.fill_betweenx(y,
x,
tr.stats.box_pos,
where=x < tr.stats.box_pos,
facecolor=c2)
ax.set_xlabel('distance (km)')
ax.set_ylim(max(y), min(y))
ax.set_ylabel('time (s)')
if moveout_model:
from rf.simple_model import load_model
model = load_model(moveout_model)
phase = profile[0].stats.moveout
slowness = profile[0].stats.slowness
pd = model.calculate_delay_times(phase=phase, slowness=slowness)
ax2 = ax.twinx()
ax.get_shared_y_axes().join(ax, ax2)
dkm = 50
if profile[0].stats.endtime - profile[0].stats.onset > 50:
dkm = 200
d1 = np.arange(20) * dkm
d2 = np.arange(100) * dkm / 5
t1 = np.interp(d1, model.z, pd)
t2 = np.interp(d2, model.z, pd)
myLocator = FixedLocator(t1)
myMinorLocator = FixedLocator(t2)
myFormatter = FixedFormatter([str(i) for i in d1])
ax2.yaxis.set_major_locator(myLocator)
ax2.yaxis.set_minor_locator(myMinorLocator)
ax2.yaxis.set_major_formatter(myFormatter)
ax2.set_ylabel('depth (km)')
ax2.set_ylim(ax.get_ylim())
if top is not None:
ax3 = fig.add_axes([0.1, 0.85, 0.8, 0.1], sharex=ax)
if top == 'hist':
left = [tr.stats.box_pos - tr.stats.box_length / 2 for tr in profile]
height = [tr.stats.num for tr in profile]
ax3.bar(left, height, widths, color='cadetblue')
plt.setp(ax3.get_xticklabels(), visible=False)
ax3.spines['top'].set_color('none')
ax3.spines['right'].set_color('none')
ax3.spines['left'].set_color('none')
ax3.xaxis.set_ticks_position('bottom')
ax3.yaxis.set_ticks_position('left')
ax3.set_yticks(ax3.get_ylim())
elif top is not None:
raise NotImplementedError("'%s' not supported for top parameter" % top)
ax.set_xlim(*xlim)
if fname:
fig.savefig(fname)
plt.close(fig)
direction="in",
width=0.5)
ax.xaxis.set_ticks_position('both')
ax.yaxis.set_ticks_position('both')
for axis in ['top', 'bottom', 'left', 'right']:
ax.spines[axis].set_linewidth(0.5)
# Deint the ticks for the x-...
xtMajor = [np.log10(10**j) for j in np.linspace(0, 3, 4)]
xtMinor = [np.log10(i * 10**j) for j in xtMajor for i in range(10)[1:10]]
xlMajor = [
r"$10^{" + str(int(i)) + "}$" if i in xtMajor else "" for i in xtMajor
]
xMajorLocator = FixedLocator(xtMajor)
xMinorLocator = FixedLocator(xtMinor)
xMajorFormatter = FixedFormatter(xlMajor)
# ... and y-axis
ytMajor = np.linspace(-21, -10, 12)
ytMinor = [np.log10(i * 10**(j)) for i in range(10)[1:] for j in ytMajor]
ylMajor = [
r"$10^{" + str(int(i)) + "}$" if i in ytMajor else "" for i in ytMajor
]
yMajorLocator = FixedLocator(ytMajor)
yMinorLocator = FixedLocator(ytMinor)
yMajorFormatter = FixedFormatter(ylMajor)
# Function to exteract the abundances for the data file
def get_abd(data, i):
i0 = i + 2
def model_evaluator(data):
#partition processed data into vectors
actualClass = data.label
predictedClass = data.predicted
probability = data.probabilty
#build a confusion matrix
cm = confusion_matrix(actualClass, predictedClass, labels = [0,1])
TruePositive = cm[1, 1]
TrueNegative = cm[0,0]
FalsePositive = cm[0,1]
FalseNegative = cm[1,0]
numberOfPositives = TruePositive + FalseNegative
numberOfNegatives = TrueNegative + FalsePositive
#calculate the Null accuracy
null_accuracy = 1 - actualClass.mean()
#define the model accuracy
model_accuracy = metrics.accuracy_score(actualClass, predictedClass)
#Generate a metrics report
report = metrics.classification_report(actualClass, predictedClass, output_dict = True)
#calculate the model performance over the null accuracy
performance_over_null = model_accuracy - null_accuracy
#Calculate the Specificity of the model
specificity = TrueNegative / (TrueNegative + FalsePositive)
#Calculate the True positive rate, false positive rate, and thresholds to plot a rock curve
fpr, tpr, thresholds = metrics.roc_curve(actualClass, probability)
#Calculate the Area under the ROC Curve
rocAuc = metrics.roc_auc_score(actualClass, probability)
#generate figure
fig = plt.figure(figsize = (12, 20))
spec = gridspec.GridSpec(ncols=2, nrows=1, wspace=0.5, width_ratios=[1, 1], figure=fig)
#plot confusion matrix in pos 0,0
confusionMatrixLabels = ['Normal Traffic', 'Intrusion']
confusionMatrixColourMap = plt.cm.Blues
confusionMatrix = fig.add_subplot(spec[0,0])
confusionMatrix.set_aspect('equal')
confusionMatrix.imshow(cm, interpolation = 'nearest', cmap = confusionMatrixColourMap)
confusionMatrix.set(ylabel ='True class', xlabel ='Predicted class')
#confusionMatrix.xlabel(labelpad=5)
confusionMatrix.set_xticks(np.arange(0,2))
formatter = FixedFormatter(['Normal Traffic', 'Intrusion'])
locator = FixedLocator([0,1])
confusionMatrix.yaxis.set_major_formatter(formatter)
confusionMatrix.yaxis.set_major_locator(locator)
confusionMatrix.xaxis.set_major_formatter(formatter)
confusionMatrix.xaxis.set_major_locator(locator)
#confusionMatrix.set_yticks(np.arange(0,2))
#confusionMatrix.set_xticklabels(np.arange(0,1), confusionMatrixLabels, fontdict = None)
tot = sum(df.label)
for i in range(cm.shape[0]):
for j in range(cm.shape[1]):
confusionMatrix.text(j, i, (format(cm[i, j])),ha ='center', va="baseline", color="white" if cm[i,j] > tot else 'black', size = 'larger')
cmLabels = ['TN', 'FP', 'FN', 'TP' ]
a = 0
for i in range(cm.shape[0]):
for j in range(cm.shape[1]):
confusionMatrix.text(j + 0.3, i + 0.4, (cmLabels[a]), ha ='center', va="baseline", color="white" if cm[i,j] > tot else 'black', size = 'larger')
if a < 4:
a += 1
a=0
for i in range(cm.shape[a]):
if a == 0:
confusionMatrix.text(j+0.8, i, ('Total:\n %d' % (numberOfNegatives)), ha ='center', va="center", color = 'black', size = 'larger')
a += 1
else:
confusionMatrix.text(j+0.8, i, ('Total:\n %d' % (numberOfPositives)), ha ='center', va="center", color = 'black', size = 'larger')
#plot roc curve in position 0,1
rocCurve = fig.add_subplot(spec[0, 1])
rocCurve.set_aspect('equal')
rocCurve.plot(fpr, tpr, color='red', lw=2, label = 'ROC area = %0.5f)' % rocAuc )
rocCurve.set(xlabel = 'True Positive Rate (Sensitivity)', ylabel = 'True Positive Rate (Sensitivity)' )
rocCurve.legend(loc="lower right")
#print report
print('The performance of this model over the null accuracy is %2.2f%%\nModel Sensitivity: %2.6f%% \nModel Specificity: %2.6f%% \nModel F1 Score: %2.6f'
% ((performance_over_null *100), (report['1']['recall']*100), (specificity*100), (report['1']['f1-score'])))
('Wind (km/h)', 'Temp (°C)', 'Rain (mm/h)'),
(wind, temp, rain),
('green', 'red', 'blue'),
([0, 90], [0, 30], [0, 15]),
)
# Plotting weather data
for ax,y_title, weather_data, color, scale in plot_data:
ax.set_ylabel(y_title)
# CAUTION: because of unexplained data parsing reasons, len(rain), len(temp) and len(wind)
# may not be equal to len(x)
# so one need to call create_x_abcissa function
x_abscissa = wp_model.create_x_abscissa(datetime_list, weather_data)
ax.plot(x_abscissa, weather_data, color=color)
ax.set_ylim(scale)
# setting shared x-axis
plt.setp([a.get_xticklabels() for a in fig.axes[:-1]], visible=False)
name_list = [elt.strftime('%a %d %b') for elt in datetime_list]
ax1.xaxis.set_major_locator(FixedLocator(x[::60]))
ax1.xaxis.set_major_formatter(FixedFormatter(name_list[::60]))
ax1.set_xlim([x[0], x[-1]])
plt.xticks(rotation=70)
ax1.set_title(title)
plt.tight_layout(pad=1, h_pad=0)
plt.savefig(f"{title}.png", format='png')
def plot_field(name,
directory,
tstamps,
samples,
nr_samples,
plot_fmt=None,
table=None,
verbose=False):
global current_plot_fmt
from matplotlib import use as muse
muse('Agg')
from matplotlib.backends.backend_agg import FigureCanvasAgg as FigureCanvas
from matplotlib.figure import Figure
from matplotlib.ticker import FuncFormatter, FixedFormatter, LinearLocator
from matplotlib.mlab import std as std_deviation
from matplotlib.mlab import mean
from time import asctime
yfont = {'fontname': 'Bitstream Vera Sans', 'color': 'r', 'fontsize': 8}
xfont = {'fontname': 'Bitstream Vera Sans', 'color': 'b', 'fontsize': 8}
titlefont = {
'fontname': 'Bitstream Vera Sans',
'color': 'g',
'fontweight': 'bold',
'fontsize': 10
}
inches = 0.00666667
width = 950 * inches
height = 680 * inches
fig = Figure(figsize=(width, height))
canvas = FigureCanvas(fig)
ax = fig.add_subplot(111)
ax.grid(False)
xtickfontsize = 5
ytickfontsize = 5
current_plot_fmt = plot_fmt
field_mean = None
plot_type = 'b-'
if current_plot_fmt == "filter_dev":
std = std_deviation(samples) * 2
if verbose:
print("filter_dev(%s) std=%d" % (name, std))
for i in range(nr_samples):
if samples[i] > std:
if verbose:
print("%s: filtering out %d" % (name, samples[i]))
samples[i] = 0
field_mean = mean(samples)
yaxis_plot_fmt = FuncFormatter(pylab_formatter)
elif current_plot_fmt == "table":
ax.grid(True)
plot_type = 'bo-'
max_value = max(samples)
without_zero = 1
if table.has_key(0):
without_zero = 0
max_value += 1
ax.yaxis.set_major_locator(LinearLocator(max_value))
tstamps = range(nr_samples)
seq = [" "] * max_value
for key in table.keys():
if key in samples:
seq[key - without_zero] = "%s(%d)" % (table[key], key)
ytickfontsize = 4
yaxis_plot_fmt = FixedFormatter(seq)
else:
field_mean = mean(samples)
yaxis_plot_fmt = FuncFormatter(pylab_formatter)
ax.plot(tstamps, samples, plot_type)
ax.set_xlabel("time", xfont)
yname = name
if field_mean:
yname += " (mean=%s)" % pylab_formatter(field_mean)
ax.set_ylabel(yname, yfont)
for label in ax.get_xticklabels():
label.set(fontsize=xtickfontsize)
for label in ax.get_yticklabels():
label.set(fontsize=ytickfontsize)
ax.yaxis.set_major_formatter(yaxis_plot_fmt)
ax.set_title("%d %s samples (%s)" % (nr_samples, name, asctime()),
titlefont)
canvas.print_figure("%s/%s.png" % (directory, name))
del fig, canvas, ax
def loghist(x, y, nbins=100,
hot=True, doclf=True, docolorbar=True, lo=0.3,
imshowargs={},
clampxlo=False, clampxlo_val=None, clampxlo_to=None,
clampxhi=False, clampxhi_val=None, clampxhi_to=None,
clampylo=False, clampylo_val=None, clampylo_to=None,
clampyhi=False, clampyhi_val=None, clampyhi_to=None,
clamp=None, clamp_to=None,
**kwargs):
#np.seterr(all='warn')
if doclf:
plt.clf()
myargs = kwargs.copy()
if not 'bins' in myargs:
myargs['bins'] = nbins
rng = kwargs.get('range', None)
x = np.array(x)
y = np.array(y)
if not (np.all(np.isfinite(x)) and np.all(np.isfinite(y))):
K = np.flatnonzero(np.isfinite(x) * np.isfinite(y))
print 'loghist: cutting to', len(K), 'of', len(x), 'finite values'
x = x[K]
y = y[K]
if clamp is True:
clamp = rng
if clamp is not None:
((clampxlo_val, clampxhi_val),(clampylo_val, clampyhi_val)) = clamp
if clamp_to is not None:
((clampxlo_to, clampxhi_to),(clampylo_to, clampyhi_to)) = clamp_to
if clampxlo:
if clampxlo_val is None:
if rng is None:
raise RuntimeError('clampxlo, but no clampxlo_val or range')
clampxlo_val = rng[0][0]
if clampxlo_val is not None:
if clampxlo_to is None:
clampxlo_to = clampxlo_val
x[x < clampxlo_val] = clampxlo_to
if clampxhi:
if clampxhi_val is None:
if rng is None:
raise RuntimeError('clampxhi, but no clampxhi_val or range')
clampxhi_val = rng[0][1]
if clampxhi_val is not None:
if clampxhi_to is None:
clampxhi_to = clampxhi_val
x[x > clampxhi_val] = clampxhi_to
if clampylo:
if clampylo_val is None:
if rng is None:
raise RuntimeError('clampylo, but no clampylo_val or range')
clampylo_val = rng[1][0]
if clampylo_val is not None:
if clampylo_to is None:
clampylo_to = clampylo_val
y[y < clampylo_val] = clampylo_to
if clampyhi:
if clampyhi_val is None:
if rng is None:
raise RuntimeError('clampyhi, but no clampyhi_val or range')
clampyhi_val = rng[1][1]
if clampyhi_val is not None:
if clampyhi_to is None:
clampyhi_to = clampyhi_val
y[y > clampyhi_val] = clampyhi_to
(H,xe,ye) = np.histogram2d(x, y, **myargs)
L = np.log10(np.maximum(lo, H.T))
myargs = dict(extent=(min(xe), max(xe), min(ye), max(ye)),
aspect='auto',
interpolation='nearest', origin='lower')
myargs.update(imshowargs)
plt.imshow(L, **myargs)
if hot:
plt.hot()
if docolorbar:
r = [np.log10(lo)] + range(int(np.ceil(L.max())))
# print 'loghist: L max', L.max(), 'r', r
plt.colorbar(ticks=r, format=FixedFormatter(
['0'] + ['%i'%(10**ri) for ri in r[1:]]))
#set_fp_err()
return H, xe, ye
def _format_ax_tick_labels(self, ax, pars):
fun_set_bounds = pars['fun_set_bounds']
bounds = pars['bounds']
bound_padding = pars['bound_padding']
major_tick_location = pars['major_tick_location']
show_major_tick_label = pars['show_major_tick_label']
num_of_minor_tick_marks = pars['num_of_minor_tick_marks']
fmt_str = pars['fmt_str']
mapping = {'set_ylim': 'yaxis', 'set_xlim': 'xaxis'}
if len(bounds) == 0:
if fun_set_bounds == 'set_ylim':
bounds = ax.get_ylim()
else:
bounds = ax.get_xlim()
if len(major_tick_location) == 0:
major_tick_location = getattr(
ax, mapping[fun_set_bounds]).get_majorticklocs()
# bound_padding = bounds[1]-major_tick_location[-1]
which_axis = mapping[fun_set_bounds]
bounds_after_add_padding = bounds[0] - bound_padding, bounds[
1] + bound_padding
major_tick_labels = []
for each in major_tick_location:
if show_major_tick_label:
major_tick_labels.append(fmt_str.format(each))
else:
major_tick_labels.append('')
minor_tick_location = []
minor_tick_labels = []
for i in range(len(major_tick_location) - 1):
start = major_tick_location[i]
end = major_tick_location[i + 1]
tick_spacing = (end - start) / (num_of_minor_tick_marks + 1)
for j in range(num_of_minor_tick_marks):
minor_tick_location.append(start + tick_spacing * (j + 1))
minor_tick_labels.append('') #not showing minor tick labels
#before starting point
if i == 0:
count = 1
while True:
if (start - count *
abs(tick_spacing)) < bounds_after_add_padding[0]:
break
else:
minor_tick_location.append(start -
abs(tick_spacing) * count)
minor_tick_labels.append(
'') #not showing minor tick labels
count = count + 1
#after the last point
elif i == (len(major_tick_location) - 2):
count = 1
while True:
if (end + count *
abs(tick_spacing)) > bounds_after_add_padding[1]:
break
else:
minor_tick_location.append(end +
abs(tick_spacing) * count)
minor_tick_labels.append(
'') #not showing minor tick labels
count = count + 1
#set limits
getattr(ax, fun_set_bounds)(*bounds_after_add_padding)
#set major tick and tick labels
getattr(ax, which_axis).set_major_formatter(
FixedFormatter(major_tick_labels))
getattr(ax, which_axis).set_major_locator(
FixedLocator(major_tick_location))
#set minor tick and tick lables (not showing the lable)
getattr(ax, which_axis).set_minor_formatter(
FixedFormatter(minor_tick_labels))
getattr(ax, which_axis).set_minor_locator(
FixedLocator(minor_tick_location))
def plotBeamWithFit(array,
center,
sideLength,
widthH,
widthV,
angle,
fileName='contourfit.png',
interpolation=True):
thisDpi = 96.
matplotlib.rcParams.update({'font.size': 8})
fig = plt.figure(figsize=(400. / thisDpi, 300. / thisDpi), dpi=thisDpi)
axis = fig.gca()
if type(sideLength) == list:
plotRange = sideLength
else:
halfSideLength = sideLength / 2.0
xStart = (center[0] - halfSideLength)
xEnd = (center[0] + halfSideLength)
yStart = (center[1] - halfSideLength)
yEnd = (center[1] + halfSideLength)
plotRange = [xStart, xEnd, yStart, yEnd]
interpolateOption = 'bicubic' if interpolation == True else 'nearest'
ims = axis.imshow(
np.fliplr(array),
cmap=plt.cm.jet,
vmin=0,
vmax=1,
# interpolation=interpolateOption, extent=plotRange)
interpolation=interpolateOption,
aspect='equal',
origin='bottom')
imageShape = array.shape
gridCenter = ((imageShape[1] / 2.0 - 1), (imageShape[0] / 2.0))
# print("plot angle: %.2f" % angle)
ellipse = Ellipse(gridCenter,
width=2 * widthH,
height=2 * widthV,
angle=angle)
ellipse.fill = False
axis.add_artist(ellipse)
fig.colorbar(ims)
axis.set_aspect('auto')
xTicks = FixedLocator([0, gridCenter[0], imageShape[1] - 1])
xTicksLabel = FixedFormatter([
"{:.2f}".format(plotRange[0]), "{:.2f}".format(center[0]),
"{:.2f}".format(plotRange[1])
])
axis.xaxis.set_major_formatter(xTicksLabel)
axis.xaxis.set_major_locator(xTicks)
axis.set_xlabel("RA")
yTicks = FixedLocator([0, gridCenter[1], imageShape[0] - 1])
yTicksLabel = FixedFormatter([
"{:.2f}".format(plotRange[3]), "{:.2f}".format(center[1]),
"{:.2f}".format(plotRange[2])
])
axis.yaxis.set_major_formatter(yTicksLabel)
axis.yaxis.set_major_locator(yTicks)
axis.yaxis.set_tick_params(rotation=90)
axis.set_ylabel("DEC")
# plt.subplots_adjust(left=0.20, right=1.00)
# axes = plt.gca()
# axes.set_xlim([x.min(),x.max()])
# axes.set_ylim([y.min(),y.max()])
# plt.xlabel('Right Ascension')
# plt.ylabel('Declination')
plt.savefig(fileName, dpi=thisDpi)
plt.close()
def stage_4(resid=None,
sky=None,
ps=None,
tim=None,
mask=None,
sourcepix=None,
filters=None,
radii_arcsec=None,
orig_catalog=None,
plots=False,
lsbcat='lsb.fits',
expnum=None,
extname=None,
pixscale=None,
**kwa):
#ok,x,y = tim.subwcs.radec2pixelxy(188.7543, 13.3847)
#print 'X,Y', x,y
fstack = np.dstack(filters)
amax = np.argmax(fstack, axis=2)
fmax = np.max(fstack, axis=2)
if plots:
sna = dict(interpolation='nearest',
origin='lower',
vmin=-2.,
vmax=32.,
cmap='gray')
plt.clf()
plt.imshow(fmax, **sna)
plt.title('Filter max')
ps.savefig()
from matplotlib.ticker import FixedFormatter
radformat = FixedFormatter(['%i' % r for r in radii_arcsec])
plt.clf()
plt.imshow(amax, interpolation='nearest', origin='lower', cmap='jet')
plt.title('Filter argmax')
plt.colorbar(ticks=np.arange(amax.max() + 1), format=radformat)
ps.savefig()
peak_amax = np.zeros_like(amax)
hot = (fmax > 10.)
blobs, nblobs = label(hot)
print 'Nblobs', nblobs
#print 'blobs max', blobs.max()
peaks = []
for blob in range(1, nblobs + 1):
I = (blobs == blob)
imax = np.argmax(fmax * I)
py, px = np.unravel_index(imax, fmax.shape)
peaks.append((px, py))
#print 'blob imax', imax
print 'px,py', px, py
print 'blob max', fmax.flat[imax]
ifilt = amax.flat[imax]
print 'blob argmax', ifilt
#print ' =', amax[py,px]
if plots:
peak_amax[I] = ifilt
px = [x for x, y in peaks]
py = [y for x, y in peaks]
if plots:
plt.clf()
plt.imshow(amax * hot,
interpolation='nearest',
origin='lower',
cmap='jet')
ax = plt.axis()
plt.plot(px, py, '+', color='w')
plt.axis(ax)
plt.title('Filter argmax')
plt.colorbar(ticks=np.arange((amax * hot).max() + 1), format=radformat)
ps.savefig()
# Total flux estimate
# fmax is S/N in the amax filter. Back out...
fluxes = []
for x, y in zip(px, py):
print 'peak', x, y
ifilt = amax[y, x]
print 'ifilt', ifilt
sn = fmax[y, x]
print 'S/N', sn
r = radii_arcsec[ifilt]
sigma = r / pixscale
knorm = 1. / (2. * np.sqrt(np.pi) * sigma)
blurred = sn * (knorm * tim.sig1)
fluxest = blurred * 2. * np.pi * (2. * sigma**2)
fluxes.append(fluxest)
fluxes = np.array(fluxes)
mags = 22.5 - 2.5 * np.log10(fluxes)
if plots:
for bg in [1, 2]:
plt.clf()
if bg == 1:
plt.imshow(peak_amax,
interpolation='nearest',
origin='lower',
cmap='jet')
else:
plt.imshow(tim.getImage(), **tim.ima)
ax = plt.axis()
plt.plot(px, py, '+', color='w', ms=10, mew=2)
# Extended sources in the catalog
if len(orig_catalog):
E = orig_catalog[np.maximum(orig_catalog.shapeexp_r,
orig_catalog.shapedev_r) >= 3.]
plt.plot(E.x, E.y, 'x', color='k', ms=10, mew=2)
for x, y, m in zip(px, py, mags):
ra, dec = tim.subwcs.pixelxy2radec(x + 1, y + 1)
plt.text(x,
y,
'%.1f (%.2f,%.2f)' % (m, ra, dec),
color='w',
ha='left',
fontsize=12)
''' evcc.fits:
https://sites.google.com/site/extendedvcc/
wget "https://sites.google.com/site/extendedvcc/table/Table2_2014_8_7.txt?attredirects=0&d=1" -O table2.txt
text2fits -S 35 -H "id_evcc id_vcc ngc ra dec ra_fiber dec_fiber delta cz_sdss cz_ned mem_evcc mem_vcc morph_pri morph_sec morph_index morph_vcc u u_err g g_err r r_err i i_err z z_err r_kron r_50" -f sssddddfffssssssffffffffffff table2.txt evcc.fits -n -
vcc.fits:
text2fits -S 27 -H "id_vcc ngc ra dec cz_sdss cz_ned mem morph_vcc u u_err g g_err r r_err i i_err z z_err r_kron r_50" -f ssddffssffffffffffff table3.txt vcc.fits -n -
'''
mydir = os.path.dirname(__file__)
fn = os.path.join(mydir, 'evcc.fits')
T = fits_table(fn)
ok, x, y = tim.subwcs.radec2pixelxy(T.ra, T.dec)
x = x[ok]
y = y[ok]
plt.plot(x, y, 'o', mec=(0, 1, 0), mfc='none', ms=50, mew=5)
fn = os.path.join(mydir, 'vcc.fits')
T = fits_table(fn)
ok, x, y = tim.subwcs.radec2pixelxy(T.ra, T.dec)
x = x[ok]
y = y[ok]
plt.plot(x, y, 'o', mec=(0, 1, 1), mfc='none', ms=50, mew=5)
plt.axis(ax)
plt.title('Peaks')
#plt.title('Filter peak argmax')
#plt.colorbar(ticks=np.arange((peak_amax*hot).max()+1), format=radformat)
ps.savefig()
LSB = fits_table()
LSB.filter = np.array([tim.band] * len(px))
LSB.expnum = np.array([expnum] * len(px)).astype(np.int32)
LSB.extname = np.array([extname] * len(px))
LSB.x = np.array(px).astype(np.int16)
LSB.y = np.array(py).astype(np.int16)
ra, dec = tim.subwcs.pixelxy2radec(LSB.x, LSB.y)
LSB.ra = ra
LSB.dec = dec
LSB.flux = fluxes.astype(np.float32)
LSB.mag = mags.astype(np.float32)
LSB.radius = np.array(radii_arcsec)[amax[py, px]].astype(np.float32)
LSB.sn = fmax[py, px].astype(np.float32)
# Apertures, radii in ARCSEC.
apertures_arcsec = np.array([0.5, 0.75, 1., 1.5, 2., 3.5, 5., 7.])
apertures = apertures_arcsec / pixscale
apxy = np.vstack((px, py)).T
import photutils
with np.errstate(divide='ignore'):
imsigma = 1.0 / tim.getInvError()
imsigma[tim.getInvError() == 0] = 0
for photimg, err, name, errname in [
(resid, imsigma, 'apflux', 'apflux_ivar'),
(tim.getImage() - sky, None, 'apimgflux', None),
((tim.getImage() - sky) * mask, None, 'apmaskedimgflux', None),
(1. - (mask * 1.), None, 'apgoodpix', None),
(sourcepix, None, 'apsources', None),
]:
apimg = []
apimgerr = []
for rad in apertures:
aper = photutils.CircularAperture(apxy, rad)
p = photutils.aperture_photometry(photimg, aper, error=err)
apimg.append(p.field('aperture_sum'))
if err is not None:
apimgerr.append(p.field('aperture_sum_err'))
ap = np.vstack(apimg).T
ap[np.logical_not(np.isfinite(ap))] = 0.
LSB.set(name, ap.astype(np.float32))
if err is not None:
apiv = 1. / (np.vstack(apimgerr).T)**2
apiv[np.logical_not(np.isfinite(apiv))] = 0.
LSB.set(errname, apiv.astype(np.float32))
LSB.cut(np.argsort(-LSB.sn))
LSB.writeto(lsbcat)
return dict(LSB=LSB, filters=None, mod=None, sky=None)
def plotConfusion(confusion, grid, title, file_name, inset=None, fudge=16):
pylab.figure()
axmain = pylab.axes()
confusion_cmap_dict = {
'red':(
(0.0, 1.0, 1.0),
(0.2, 1.0, 0.5), # Missing to Correct Negative
(0.4, 0.5, 1.0), # Correct Negative to False Alarm
(0.6, 1.0, 1.0), # False Alarm to Miss
(0.8, 1.0, 0.0), # Miss to Hit
(1.0, 0.0, 0.0),
),
'green':(
(0.0, 1.0, 1.0),
(0.2, 1.0, 0.5),
(0.4, 0.5, 0.0),
(0.6, 0.0, 1.0),
(0.8, 1.0, 0.5),
(1.0, 0.5, 0.5),
),
'blue':(
(0.0, 1.0, 1.0),
(0.2, 1.0, 0.5),
(0.4, 0.5, 1.0),
(0.6, 1.0, 0.0),
(0.8, 0.0, 0.0),
(1.0, 0.0, 0.0),
),
}
confusion_cmap = LinearSegmentedColormap('confusion', confusion_cmap_dict, 256)
tick_labels = [ "Missing", "Correct\nNegative", "False\nAlarm", "Miss", "Hit" ]
min_label = -1
xs, ys = grid.getXY()
pylab.pcolormesh(xs, ys, confusion, cmap=confusion_cmap, vmin=min_label, vmax=(min_label + len(tick_labels) - 1))
tick_locs = np.linspace(-1, min_label + len(tick_labels) - 2, len(tick_labels))
tick_locs += (tick_locs[1] - tick_locs[0]) / 2
bar = pylab.colorbar()
bar.locator = FixedLocator(tick_locs)
bar.formatter = FixedFormatter(tick_labels)
pylab.setp(pylab.getp(bar.ax, 'ymajorticklabels'), fontsize='large')
bar.update_ticks()
grid.drawPolitical()
if inset:
lb_y, lb_x = [ b.start for b in inset ]
ub_y, ub_x = [ b.stop + fudge for b in inset ]
inset_exp = (slice(lb_y, ub_y), slice(lb_x, ub_x))
axins = zoomed_inset_axes(pylab.gca(), 2, loc=4)
pylab.sca(axins)
pylab.pcolormesh(xs[inset_exp], ys[inset_exp], confusion[inset_exp], cmap=confusion_cmap, vmin=0, vmax=3)
grid.drawPolitical()
pylab.xlim([lb_x * gs_x, (ub_x - 1) * gs_x])
pylab.ylim([lb_y * gs_y, (ub_y - 1) * gs_y])
mark_inset(axmain, axins, loc1=1, loc2=3, fc='none', ec='k')
pylab.sca(axmain)
pylab.suptitle(title)
pylab.savefig(file_name)
pylab.close()
return
