def testWsf(self): fileName = 'ucsb_100mK_24db_1.txt' wsf = WideSweepFile(fileName) wsf.fitSpline() wsf.findPeaks() fig,ax = plt.subplots(2) ax[0].plot(wsf.x,wsf.mag,label='mag') ax[0].set_xlabel("frequency") ax[0].set_ylabel("magnitude") ax[0].legend() ax[1].plot(wsf.x,wsf.mag,label='mag') ax[1].set_xlabel("frequency") ax[1].set_ylabel("magnitude") ax[1].legend().get_frame().set_alpha(0.5) xmin = 5.0 xmax = 5.012 ax[1].set_xlim(xmin, xmax) for peak in wsf.peaks: x = wsf.x[peak] if x > xmin and x < xmax: ax[1].axvline(x=x,color='r') plt.savefig("testWsf.png")
def testWsf2(self): fileName = 'ucsb_100mK_24db_1.txt' wsf = WideSweepFile(fileName) var = wsf.mag.var() m = len(wsf.mag) #splineS = m*var/20.0 #print "var,m,splineS=",var,m,splineS #splineK = 5 #wsf.fitSpline(splineS=splineS,splineK=splineK) wn = 0.01 wsf.fitFilter(wn=wn) m = 2 wsf.findPeaks(m=m) fig,ax = plt.subplots(2) ax[0].plot(wsf.x,wsf.mag,label='mag') ax[0].plot(wsf.x,wsf.baseline,label='baseline') ax[0].set_xlabel("frequency") ax[0].set_ylabel("magnitude") ax[0].legend() xmin = 3.25 xmax = 3.27 ax[0].set_xlim(xmin, xmax) for peak in wsf.peaks: x = wsf.x[peak] if x > xmin and x < xmax: ax[0].axvline(x=x,color='r') ax[0].set_title("filter wn=%f findPeaks m=%f"%(wn,m)) ax[1].plot(wsf.x,wsf.mag,label='mag') ax[1].plot(wsf.x,wsf.baseline,label='baseline') ax[1].set_xlabel("frequency") ax[1].set_ylabel("magnitude") ax[1].legend().get_frame().set_alpha(0.5) xmin = 5.0 xmax = 5.012 ax[1].set_xlim(xmin, xmax) for peak in wsf.peaks: x = wsf.x[peak] if x > xmin and x < xmax: ax[1].axvline(x=x,color='r') plt.savefig("testWsf2.png")
class WideAna(QMainWindow): def __init__(self, parent=None, plotFunc=None, title='', separateProcess=False, image=None, showMe=True, initialFile=None, flNum=None): self.parent = parent if self.parent == None: self.app = QApplication([]) super(WideAna, self).__init__(parent) self.fitParams = { "filter": { "order": 4, "rs": 40, "wn": 0.1 }, 'spline': { "splineS": 1, "splineK": 3 } } self.initialFile = initialFile self.baseFile = ('.').join(initialFile.split('.')[:-1]) self.goodFile = self.baseFile + "-freqs-good.txt" self.allFile = self.baseFile + "-freqs-all.txt" if os.path.exists(self.goodFile): self.goodFile = self.goodFile + time.strftime("-%Y-%m-%d-%H-%M-%S") #shutil.copy(self.goodFile,self.goodFile+time.strftime("-%Y-%m-%d-%H-%M-%S")) if os.path.exists(self.allFile): self.allFile = self.allFile + time.strftime("-%Y-%m-%d-%H-%M-%S") self.pdfFile = self.baseFile + "-good.pdf" self.fitLineEdits = {} self.flNum = flNum self.fitLabels = {} self.splineS = 1 self.splineK = 3 self.setWindowTitle(title) self.plotFunc = plotFunc self.create_main_frame(title) if plotFunc != None: plotFunc(fig=self.fig, axes=self.axes) if showMe == True: self.show() self.load_file(initialFile) # make the PDF file if not os.path.exists(self.pdfFile): print "Create overview PDF file:", self.pdfFile self.wsf.createPdf(self.pdfFile) else: print "Overview PDF file already on disk:", self.pdfFile # plot the first segment self.deltaXDisplay = 0.100 # display width in GHz self.zoomFactor = 2.0 self.segment = 0 self.calcXminXmax() self.plotSegment() print "Ready to add and delete peaks." def draw(self): self.fig.canvas.draw() def on_key_press(self, event): #print "WideAna.on_key_press: event.key=",event.key if event.key == "right" or event.key == 'r': self.segmentIncrement(None, 0.1) return elif event.key == "left" or event.key == 'l': self.segmentDecrement(None, 0.1) return elif event.key == "up" or event.key == '+': self.zoom(1.25) return elif event.key == "down" or event.key == '-': self.zoom(0.8) return self.on_key_or_button(event, event.key) def on_button_press(self, event): self.on_key_or_button(event, event.button) def on_key_or_button(self, event, pressed): xdata = getattr(event, 'xdata', None) if xdata is not None: ind = np.searchsorted(self.wsf.x, xdata) xFound = self.wsf.x[ind] indPk = np.searchsorted(self.wsf.pk, ind) xPkFound0 = self.wsf.x[self.wsf.pk[indPk - 1]] xPkFound1 = self.wsf.x[self.wsf.pk[indPk]] if abs(xPkFound0 - xdata) < abs(xPkFound1 - xdata): bestIndex = indPk - 1 else: bestIndex = indPk bestWsfIndex = self.wsf.pk[bestIndex] bestX = self.wsf.x[bestWsfIndex] if pressed == "d": #if self.peakMask[bestWsfIndex]: #self.peakMask[bestWsfIndex] = False self.goodPeakMask[ bestWsfIndex - 7:bestWsfIndex + 7] = False # larger area of indicies to pinpoint false resonator location self.badPeakMask[ bestWsfIndex - 7:bestWsfIndex + 7] = False # larger area of indicies to pinpoint false resonator location self.setCountLabel() self.replot() self.writeToGoodFile() if pressed == "a": if not self.goodPeakMask[bestWsfIndex]: self.goodPeakMask[bestWsfIndex] = True self.badPeakMask[bestWsfIndex] = False self.setCountLabel() self.replot() self.writeToGoodFile() self.writeToAllFile() if pressed == "s": if not self.badPeakMask[bestWsfIndex]: self.badPeakMask[bestWsfIndex] = True self.goodPeakMask[bestWsfIndex] = False self.setCountLabel() self.replot() self.writeToGoodFile() self.writeToAllFile() def replot(self): xlim = self.axes.set_xlim() self.xMin = xlim[0] self.xMax = xlim[1] self.plotSegment() def create_main_frame(self, title): self.main_frame = QWidget() # Create the mpl Figure and FigCanvas objects. self.dpi = 100 self.fig = Figure((7, 5), dpi=self.dpi) self.canvas = FigureCanvas(self.fig) self.canvas.setParent(self.main_frame) self.canvas.setFocusPolicy(Qt.StrongFocus) self.canvas.setFocus() self.axes = self.fig.add_subplot(111) self.fig.canvas.mpl_connect('key_press_event', self.on_key_press) self.fig.canvas.mpl_connect('button_press_event', self.on_button_press) # Create the segment slider self.segmentSlider = QtGui.QSlider(Qt.Horizontal, self) self.segmentSlider.setToolTip("Slide to change segment") self.segmentMax = 100000.0 self.segmentSlider.setRange(0, int(self.segmentMax)) self.segmentSlider.setFocusPolicy(Qt.NoFocus) self.segmentSlider.setGeometry(30, 40, 100, 30) self.segmentSlider.valueChanged[int].connect(self.changeSegmentValue) # Create the left and right buttons segmentDecrement = QtGui.QPushButton('<', self) segmentDecrement.setToolTip("Back to previous segment") segmentDecrement.clicked[bool].connect(self.segmentDecrement) segmentIncrement = QtGui.QPushButton('>', self) segmentIncrement.setToolTip("Forward to next segment") segmentIncrement.clicked[bool].connect(self.segmentIncrement) # create display mode button self.yDisplay = QtGui.QPushButton("raw") self.yDisplay.setToolTip( "Toggle y axis: raw data or difference=raw-baseline") self.yDisplay.setCheckable(True) self.yDisplay.clicked[bool].connect(self.yDisplayClicked) # create information boxes self.instructionsLabel = QtGui.QLabel() self.instructionsLabel.setText( "ADD peak: a; REMOVE peak: d ZOOM: +/- SCAN l/r ") self.countLabel = QtGui.QLabel() self.countLabel.setText("count label") self.inputLabel = QtGui.QLabel() self.inputLabel.setText("Input File:%s" % self.initialFile) self.goodLabel = QtGui.QLabel() self.goodLabel.setText("Good File:%s" % self.goodFile) # create controls for baseline fitting #self.baseline = QtGui.QPushButton("filter") #self.baseline.setCheckable(True) #self.baseline.clicked[bool].connect(self.baselineClicked) # Create the navigation toolbar, tied to the canvas self.mpl_toolbar = NavigationToolbar(self.canvas, self.main_frame) # Do the layout # segment box segmentBox = QHBoxLayout() segmentBox.addWidget(segmentDecrement) segmentBox.addWidget(self.segmentSlider) segmentBox.addWidget(segmentIncrement) segmentBox.addWidget(self.yDisplay) # baseline box #self.baselineBox = QHBoxLayout() #self.updateBaselineBox() # info box self.infoBox = QVBoxLayout() self.infoBox.addWidget(self.inputLabel) self.infoBox.addWidget(self.goodLabel) self.infoBox.addWidget(self.countLabel) self.infoBox.addWidget(self.instructionsLabel) # entire box vbox = QVBoxLayout() vbox.addLayout(self.infoBox) vbox.addLayout(segmentBox) #vbox.addLayout(self.baselineBox) vbox.addWidget(self.canvas) vbox.addWidget(self.mpl_toolbar) self.main_frame.setLayout(vbox) self.setCentralWidget(self.main_frame) def updateBaselineBox(self): for i in range(self.baselineBox.count()): item = self.baselineBox.itemAt(i) self.baselineBox.removeItem(item) mode = str(self.baseline.text()) self.baseline.setFixedSize(70, 40) self.baselineBox.addWidget(self.baseline) keys = self.fitParams[mode] def load_file(self, fileName): self.wsf = WideSweepFile(fileName) #self.wsf.fitSpline(splineS=1.0, splineK=1) self.wsf.fitFilter(wn=0.01) self.wsf.findPeaks(m=2) self.goodPeakMask = np.zeros(len(self.wsf.x), dtype=np.bool) self.badPeakMask = np.zeros(len(self.wsf.x), dtype=np.bool) self.collMask = np.zeros(len(self.wsf.x), dtype=np.bool) if os.path.isfile( self.baseFile + "-ml.txt" ): # update: use machine learning peak loacations if they've been made print 'loading peak location predictions from', self.baseFile + "-ml.txt" peaks = np.loadtxt(self.baseFile + "-ml.txt") peaks = map(int, peaks) else: peaks = self.wsf.peaks #coll_thresh = self.wsf.x[0] dist = abs(np.roll(peaks, -1) - peaks) #colls = np.delete(peaks,np.where(dist>=9)) colls = [] diff, coll_thresh = 0, 0 while diff <= 5e-4: diff = self.wsf.x[coll_thresh] - self.wsf.x[0] coll_thresh += 1 print coll_thresh for i in range(len(peaks)): if dist[i] < coll_thresh: if self.wsf.mag[peaks[i + 1]] - self.wsf.mag[peaks[i]] > 1.5: colls.append(peaks[i + 1]) #print 'for ', self.wsf.x[peaks[i]], 'chosing the one before' else: colls.append(peaks[i]) #print 'for ', self.wsf.x[peaks[i]], 'chosing the this one' print colls if colls != []: #colls=np.array(map(int,colls)) self.collMask[ colls] = True # update: so unidentified peaks can be identified as unusable peaks = np.delete( peaks, colls) #remove collisions (peaks < 0.5MHz apart = < 9 steps apart) #peaks = np.delete(peaks,np.where(dist<9)) #remove collisions (peaks < 0.5MHz apart = < 9 steps apart) self.goodPeakMask[peaks] = True self.setCountLabel() self.writeToGoodFile() def setCountLabel(self): self.countLabel.setText("Number of good peaks = %d" % self.goodPeakMask.sum()) def writeToGoodFile(self): gf = open(self.goodFile, 'wb') id = (self.flNum - 1) * 2000 for index in range(len(self.goodPeakMask)): if self.goodPeakMask[index]: line = "%8d %12d %16.7f\n" % (id, index, self.wsf.x[index]) gf.write(line) id += 1 elif self.badPeakMask[index]: id += 1 gf.close() def writeToAllFile(self): af = open(self.allFile, 'wb') id = (self.flNum - 1) * 2000 for index in range(len(self.goodPeakMask)): if self.goodPeakMask[index] or self.badPeakMask[index]: line = "%8d %12d %16.7f\n" % (id, index, self.wsf.x[index]) af.write(line) id += 1 af.close() # deal with zooming and plotting one segment def zoom(self, zoom): self.zoomFactor *= zoom self.calcXminXmax() self.plotSegment() def changeSegmentValue(self, value): self.segment = value self.calcXminXmax() self.plotSegment() def segmentDecrement(self, value, amount=0.9): wsfDx = self.wsf.x[-1] - self.wsf.x[0] plotDx = self.xMax - self.xMin dsdx = self.segmentMax / wsfDx ds = amount * dsdx * plotDx self.segment = max(0, self.segment - ds) self.segmentSlider.setSliderPosition(self.segment) def segmentIncrement(self, value, amount=0.9): wsfDx = self.wsf.x[-1] - self.wsf.x[0] plotDx = self.xMax - self.xMin dsdx = self.segmentMax / wsfDx ds = amount * dsdx * plotDx self.segment = min(self.segmentMax, self.segment + ds) self.segmentSlider.setSliderPosition(self.segment) def calcXminXmax(self): xMiddle = self.wsf.x[0] + \ (self.segment/self.segmentMax)*(self.wsf.x[-1]-self.wsf.x[0]) dx = self.deltaXDisplay / self.zoomFactor self.xMin = xMiddle - dx / 2.0 self.xMax = xMiddle + dx / 2.0 def plotSegment(self): ydText = self.yDisplay.text() if self.wsf != None: if ydText == "raw": yPlot = 20 * np.log10(self.wsf.mag) yName = "log-magnitude" else: yPlot = self.wsf.mag - self.wsf.baseline yName = "log(mag-baseline)" stride = self.wsf.data1.shape[0] / self.segmentMax # plot all values and then set xmin and xmax to show this segment self.axes.clear() self.axes.plot(self.wsf.x, yPlot, label=yName) for x in self.wsf.x[self.goodPeakMask]: if x > self.xMin and x < self.xMax: self.axes.axvline(x=x, color='g') self.axes.axvline(x=x + 0.00025, color='g', linestyle='-.', linewidth=0.5) self.axes.axvline(x=x - 0.00025, color='g', linestyle='-.', linewidth=0.5) for c in self.wsf.x[self.collMask]: if c > self.xMin and c < self.xMax: self.axes.axvline(x=c, color='g') for x in self.wsf.x[self.badPeakMask]: if x > self.xMin and x < self.xMax: self.axes.axvline(x=x, color='r') self.axes.axvline(x=x + 0.00025, color='r', linestyle='-.', linewidth=0.5) self.axes.axvline(x=x - 0.00025, color='r', linestyle='-.', linewidth=0.5) self.axes.set_xlim((self.xMin, self.xMax)) self.axes.set_title("segment=%.1f/%.1f" % (self.segment, self.segmentMax)) #self.axes.legend().get_frame().set_alpha(0.5) self.draw() def yDisplayClicked(self, value): if value: self.yDisplay.setText("diff") else: self.yDisplay.setText("raw") self.replot() def baselineClicked(self, value): if value: self.baseline.setText("spline") else: self.baseline.setText("filter") self.updateBaselineBox() def show(self): super(WideAna, self).show() if self.parent == None: self.app.exec_()
class WideAna(QMainWindow): def __init__(self, parent=None,plotFunc=None,title='',separateProcess=False, image=None,showMe=True, initialFile=None): self.parent = parent if self.parent == None: self.app = QApplication([]) super(WideAna,self).__init__(parent) self.fitParams = { "filter":{"order":4,"rs":40,"wn":0.1}, 'spline':{"splineS":1,"splineK":3}} self.initialFile = initialFile self.baseFile = ('.').join(initialFile.split('.')[:-1]) self.goodFile = self.baseFile+"-good.txt" if os.path.exists(self.goodFile): self.goodFile = self.goodFile+time.strftime("-%Y-%m-%d-%H-%M-%S") #shutil.copy(self.goodFile,self.goodFile+time.strftime("-%Y-%m-%d-%H-%M-%S")) self.pdfFile = self.baseFile+"-good.pdf" self.fitLineEdits = {} self.fitLabels = {} self.splineS = 1 self.splineK = 3 self.setWindowTitle(title) self.plotFunc = plotFunc self.create_main_frame(title) if plotFunc != None: plotFunc(fig=self.fig,axes=self.axes) if showMe == True: self.show() self.load_file(initialFile) # make the PDF file if not os.path.exists(self.pdfFile): print "Create overview PDF file:",self.pdfFile self.wsf.createPdf(self.pdfFile) else: print "Overview PDF file already on disk:",self.pdfFile # plot the first segment self.deltaXDisplay = 0.100 # display width in GHz self.zoomFactor = 1.0 self.segment = 0 self.calcXminXmax() self.plotSegment() print "Ready to add and delete peaks." def draw(self): self.fig.canvas.draw() def on_key_press(self,event): #print "WideAna.on_key_press: event.key=",event.key if event.key == "right" or event.key == 'r': self.segmentIncrement(None,0.1) return elif event.key == "left" or event.key == 'l': self.segmentDecrement(None,0.1) return elif event.key == "up" or event.key == '+': self.zoom(1.25) return elif event.key == "down" or event.key == '-': self.zoom(0.8) return self.on_key_or_button(event, event.key) def on_button_press(self,event): self.on_key_or_button(event,event.button) def on_key_or_button(self, event, pressed): xdata = getattr(event, 'xdata', None) if xdata is not None: ind = np.searchsorted(self.wsf.x, xdata) xFound = self.wsf.x[ind] indPk = np.searchsorted(self.wsf.pk, ind) xPkFound0 = self.wsf.x[self.wsf.pk[indPk-1]] xPkFound1 = self.wsf.x[self.wsf.pk[indPk]] if abs(xPkFound0-xdata) < abs(xPkFound1-xdata): bestIndex = indPk-1 else: bestIndex = indPk bestWsfIndex = self.wsf.pk[bestIndex] bestX = self.wsf.x[bestWsfIndex] if pressed == "d": #if self.peakMask[bestWsfIndex]: #self.peakMask[bestWsfIndex] = False self.peakMask[bestWsfIndex-7:bestWsfIndex+7] = False # larger area of indicies to pinpoint false resonator location self.setCountLabel() self.replot() self.writeToGoodFile() if pressed == "a": if not self.peakMask[bestWsfIndex]: self.peakMask[bestWsfIndex] = True self.setCountLabel() self.replot() self.writeToGoodFile() def replot(self): xlim = self.axes.set_xlim() self.xMin=xlim[0] self.xMax=xlim[1] self.plotSegment() def create_main_frame(self,title): self.main_frame = QWidget() # Create the mpl Figure and FigCanvas objects. self.dpi = 100 self.fig = Figure((7, 5), dpi=self.dpi) self.canvas = FigureCanvas(self.fig) self.canvas.setParent(self.main_frame) self.canvas.setFocusPolicy(Qt.StrongFocus) self.canvas.setFocus() self.axes = self.fig.add_subplot(111) self.fig.canvas.mpl_connect('key_press_event',self.on_key_press) self.fig.canvas.mpl_connect('button_press_event',self.on_button_press) # Create the segment slider self.segmentSlider = QtGui.QSlider(Qt.Horizontal, self) self.segmentSlider.setToolTip("Slide to change segment") self.segmentMax = 100000.0 self.segmentSlider.setRange(0,int(self.segmentMax)) self.segmentSlider.setFocusPolicy(Qt.NoFocus) self.segmentSlider.setGeometry(30,40,100,30) self.segmentSlider.valueChanged[int].connect(self.changeSegmentValue) # Create the left and right buttons segmentDecrement = QtGui.QPushButton('<',self) segmentDecrement.setToolTip("Back to previous segment") segmentDecrement.clicked[bool].connect(self.segmentDecrement) segmentIncrement = QtGui.QPushButton('>',self) segmentIncrement.setToolTip("Forward to next segment") segmentIncrement.clicked[bool].connect(self.segmentIncrement) # create display mode button self.yDisplay = QtGui.QPushButton("raw") self.yDisplay.setToolTip("Toggle y axis: raw data or difference=raw-baseline") self.yDisplay.setCheckable(True) self.yDisplay.clicked[bool].connect(self.yDisplayClicked) # create information boxes self.instructionsLabel = QtGui.QLabel() self.instructionsLabel.setText("ADD peak: a; REMOVE peak: d ZOOM: +/- SCAN l/r ") self.countLabel = QtGui.QLabel() self.countLabel.setText("count label") self.inputLabel = QtGui.QLabel() self.inputLabel.setText("Input File:%s"%self.initialFile) self.goodLabel = QtGui.QLabel() self.goodLabel.setText("Good File:%s"%self.goodFile) # create controls for baseline fitting #self.baseline = QtGui.QPushButton("filter") #self.baseline.setCheckable(True) #self.baseline.clicked[bool].connect(self.baselineClicked) # Create the navigation toolbar, tied to the canvas self.mpl_toolbar = NavigationToolbar(self.canvas, self.main_frame) # Do the layout # segment box segmentBox = QHBoxLayout() segmentBox.addWidget(segmentDecrement) segmentBox.addWidget(self.segmentSlider) segmentBox.addWidget(segmentIncrement) segmentBox.addWidget(self.yDisplay) # baseline box #self.baselineBox = QHBoxLayout() #self.updateBaselineBox() # info box self.infoBox = QVBoxLayout() self.infoBox.addWidget(self.inputLabel) self.infoBox.addWidget(self.goodLabel) self.infoBox.addWidget(self.countLabel) self.infoBox.addWidget(self.instructionsLabel) # entire box vbox = QVBoxLayout() vbox.addLayout(self.infoBox) vbox.addLayout(segmentBox) #vbox.addLayout(self.baselineBox) vbox.addWidget(self.canvas) vbox.addWidget(self.mpl_toolbar) self.main_frame.setLayout(vbox) self.setCentralWidget(self.main_frame) def updateBaselineBox(self): for i in range(self.baselineBox.count()): item = self.baselineBox.itemAt(i) self.baselineBox.removeItem(item) mode = str(self.baseline.text()) self.baseline.setFixedSize(70,40) self.baselineBox.addWidget(self.baseline) keys = self.fitParams[mode] def load_file(self, fileName): self.wsf = WideSweepFile(fileName) #self.wsf.fitSpline(splineS=1.0, splineK=1) self.wsf.fitFilter(wn=0.01) self.wsf.findPeaks(m=2) self.peakMask = np.zeros(len(self.wsf.x),dtype=np.bool) self.collMask = np.zeros(len(self.wsf.x),dtype=np.bool) if os.path.isfile(self.baseFile+"-ml.txt"): # update: use machine learning peak loacations if they've been made print 'loading peak location predictions from', self.baseFile+"-ml.txt" peaks = np.loadtxt(self.baseFile+"-ml.txt") peaks = map(int,peaks) else: peaks = self.wsf.peaks #coll_thresh = self.wsf.x[0] dist = abs(np.roll(peaks, -1) - peaks) #colls = np.delete(peaks,np.where(dist>=9)) colls=[] diff, coll_thresh = 0, 0 while diff <= 5e-4: diff = self.wsf.x[coll_thresh]-self.wsf.x[0] coll_thresh += 1 print coll_thresh for i in range(len(peaks)): if dist[i]<coll_thresh: if self.wsf.mag[peaks[i+1]] - self.wsf.mag[peaks[i]] > 1.5: colls.append(peaks[i+1]) #print 'for ', self.wsf.x[peaks[i]], 'chosing the one before' else: colls.append(peaks[i]) #print 'for ', self.wsf.x[peaks[i]], 'chosing the this one' print colls if colls != []: #colls=np.array(map(int,colls)) self.collMask[colls] = True # update: so unidentified peaks can be identified as unusable peaks = np.delete(peaks,colls) #remove collisions (peaks < 0.5MHz apart = < 9 steps apart) #peaks = np.delete(peaks,np.where(dist<9)) #remove collisions (peaks < 0.5MHz apart = < 9 steps apart) self.peakMask[peaks] = True self.setCountLabel() self.writeToGoodFile() def setCountLabel(self): self.countLabel.setText("Number of peaks = %d"%self.peakMask.sum()) def writeToGoodFile(self): gf = open(self.goodFile,'wb') id = 0 for index in range(len(self.peakMask)): if self.peakMask[index]: line = "%8d %12d %16.7f\n"%(id,index,self.wsf.x[index]) gf.write(line) id += 1 gf.close() # deal with zooming and plotting one segment def zoom(self,zoom): self.zoomFactor *= zoom self.calcXminXmax() self.plotSegment() def changeSegmentValue(self,value): self.segment = value self.calcXminXmax() self.plotSegment() def segmentDecrement(self, value, amount=0.9): wsfDx = self.wsf.x[-1]-self.wsf.x[0] plotDx = self.xMax-self.xMin dsdx = self.segmentMax / wsfDx ds = amount * dsdx * plotDx self.segment = max(0,self.segment-ds) self.segmentSlider.setSliderPosition(self.segment) def segmentIncrement(self, value, amount=0.9): wsfDx = self.wsf.x[-1]-self.wsf.x[0] plotDx = self.xMax-self.xMin dsdx = self.segmentMax / wsfDx ds = amount * dsdx * plotDx self.segment = min(self.segmentMax,self.segment+ds) self.segmentSlider.setSliderPosition(self.segment) def calcXminXmax(self): xMiddle = self.wsf.x[0] + \ (self.segment/self.segmentMax)*(self.wsf.x[-1]-self.wsf.x[0]) dx = self.deltaXDisplay/self.zoomFactor self.xMin = xMiddle-dx/2.0 self.xMax = xMiddle+dx/2.0 def plotSegment(self): ydText = self.yDisplay.text() if self.wsf != None: if ydText == "raw": yPlot = self.wsf.mag yName = "magnitude" else: yPlot = self.wsf.mag-self.wsf.baseline yName = "mag-baseline" stride = self.wsf.data1.shape[0]/self.segmentMax # plot all values and then set xmin and xmax to show this segment self.axes.clear() self.axes.plot(self.wsf.x, yPlot, label=yName) for x in self.wsf.x[self.peakMask]: if x > self.xMin and x < self.xMax: self.axes.axvline(x=x,color='r') for c in self.wsf.x[self.collMask]: if c > self.xMin and c < self.xMax: self.axes.axvline(x=c,color='g') self.axes.set_xlim((self.xMin,self.xMax)) self.axes.set_title("segment=%.1f/%.1f"%(self.segment,self.segmentMax)) self.axes.legend().get_frame().set_alpha(0.5) self.draw() def yDisplayClicked(self, value): if value: self.yDisplay.setText("diff") else: self.yDisplay.setText("raw") self.replot() def baselineClicked(self,value): if value: self.baseline.setText("spline") else: self.baseline.setText("filter") self.updateBaselineBox() def show(self): super(WideAna,self).show() if self.parent == None: self.app.exec_()
class WideAna(QMainWindow): def __init__(self, parent=None, plotFunc=None, title='', separateProcess=False, image=None, showMe=True, initialFile=None, flNum=None): self.coll_thresh = 1.E-4 #100 kHz self.parent = parent if self.parent == None: self.app = QApplication([]) super(WideAna, self).__init__(parent) self.fitParams = { "filter": { "order": 4, "rs": 40, "wn": 0.1 }, 'spline': { "splineS": 1, "splineK": 3 } } self.initialFile = initialFile self.baseFile = ('.').join(initialFile.split('.')[:-1]) self.goodFile = self.baseFile + "-freqs-good.txt" self.allFile = self.baseFile + "-freqs-all.txt" self.pdfFile = self.baseFile + "-good.pdf" self.fitLineEdits = {} self.flNum = flNum self.fitLabels = {} self.splineS = 1 self.splineK = 3 self.setWindowTitle(title) self.plotFunc = plotFunc self.create_main_frame(title) if plotFunc != None: plotFunc(fig=self.fig, axes=self.axes) if showMe == True: self.show() self.load_file(initialFile) # make the PDF file if os.path.exists(self.goodFile) and os.path.exists(self.allFile): # self.goodFile = self.goodFile+time.strftime("-%Y-%m-%d-%H-%M-%S") #shutil.copy(self.goodFile,self.goodFile+time.strftime("-%Y-%m-%d-%H-%M-%S")) self.load_old_data(self.goodFile, self.allFile) # if os.path.exists(self.allFile): # self.allFile = self.allFile+time.strftime("-%Y-%m-%d-%H-%M-%S") if not os.path.exists(self.pdfFile): print "Create overview PDF file:", self.pdfFile self.wsf.createPdf(self.pdfFile) else: print "Overview PDF file already on disk:", self.pdfFile self.seggrouper(self.wsf.x) # plot the first segment self.deltaXDisplay = 0.100 # display width in GHz self.zoomFactor = 2.0 self.segment = 0 self.calcXminXmax() self.plotSegment() self.ylim = self.axes.get_ylim() print "Ready to add and delete peaks." def load_old_data(self, goodFile, allFile): print '\nWARNING: loading old data from \n' + str( goodFile) + '\nand\n' + str(allFile) print '\nThese files WILL BE MODIFIED! Exit the program if you want to abort.\n' old_good_data = np.loadtxt(goodFile) if len(old_good_data) == 0: pass else: old_good_ind = old_good_data[:, 1].astype(int) #get the old indexes self.goodPeakMask[old_good_ind] = True old_all_data = np.loadtxt(allFile) if len(old_all_data) == 0: return else: old_all_ind = old_all_data[:, 1].astype(int) allMask = np.zeros(len(self.wsf.x), dtype=np.bool) allMask[old_all_ind] = True self.badPeakMask = np.logical_xor(self.goodPeakMask, allMask) def draw(self): self.fig.canvas.draw() def on_key_press(self, event): #print "WideAna.on_key_press: event.key=",event.key if event.key == "right" or event.key == 'r': self.segmentIncrement(None, 0.1) return elif event.key == "left" or event.key == 'l': self.segmentDecrement(None, 0.1) return elif event.key == "up" or event.key == '+': self.zoom(1.25) return elif event.key == "down" or event.key == '-': self.zoom(0.8) return self.on_key_or_button(event, event.key) def on_button_press(self, event): self.on_key_or_button(event, event.button) def on_key_or_button(self, event, pressed): xdata = getattr(event, 'xdata', None) if xdata is not None: ind = np.argmin(np.abs(self.wsf.x - xdata)) xFound = self.wsf.x[ind] indPk = np.searchsorted(self.wsf.pk, ind) xPkFound0 = self.wsf.x[self.wsf.pk[indPk - 1]] xPkFound1 = self.wsf.x[self.wsf.pk[indPk]] if abs(xPkFound0 - xdata) < abs(xPkFound1 - xdata): bestIndex = indPk - 1 else: bestIndex = indPk bestWsfIndex = self.wsf.pk[bestIndex] bestX = self.wsf.x[bestWsfIndex] if pressed == "d": #self.goodPeakMask[bestWsfIndex-7:bestWsfIndex+7] = False # larger area of indicies to pinpoint false resonator location #self.badPeakMask[bestWsfIndex-7:bestWsfIndex+7] = False # larger area of indicies to pinpoint false resonator location self.goodPeakMask[np.where( np.logical_and(self.wsf.x > (xFound - self.coll_thresh), self.wsf.x < (xFound + self.coll_thresh)))] = False self.badPeakMask[np.where( np.logical_and(self.wsf.x > (xFound - self.coll_thresh), self.wsf.x < (xFound + self.coll_thresh)))] = False self.setCountLabel() self.replot() self.writeToGoodFile() self.writeToAllFile() if pressed == "a": if not self.goodPeakMask[bestWsfIndex]: self.goodPeakMask[bestWsfIndex] = True self.badPeakMask[bestWsfIndex] = False self.setCountLabel() self.replot() self.writeToGoodFile() self.writeToAllFile() if pressed == "s": if not self.badPeakMask[bestWsfIndex]: self.badPeakMask[bestWsfIndex] = True self.goodPeakMask[bestWsfIndex] = False self.setCountLabel() self.replot() self.writeToGoodFile() self.writeToAllFile() def replot(self): xlim = self.axes.set_xlim() self.ylim = self.axes.get_ylim() self.xMin = xlim[0] self.xMax = xlim[1] self.plotSegment() def create_main_frame(self, title): self.main_frame = QWidget() # Create the mpl Figure and FigCanvas objects. self.dpi = 100 self.fig = Figure((7, 5), dpi=self.dpi) self.canvas = FigureCanvas(self.fig) self.canvas.setParent(self.main_frame) self.canvas.setFocusPolicy(Qt.StrongFocus) self.canvas.setFocus() self.axes = self.fig.add_subplot(111) self.fig.canvas.mpl_connect('key_press_event', self.on_key_press) self.fig.canvas.mpl_connect('button_press_event', self.on_button_press) # Create the segment slider self.segmentSlider = QtGui.QSlider(Qt.Horizontal, self) self.segmentSlider.setToolTip("Slide to change segment") self.segmentMax = 100000.0 self.segmentSlider.setRange(0, int(self.segmentMax)) self.segmentSlider.setFocusPolicy(Qt.NoFocus) self.segmentSlider.setGeometry(30, 40, 100, 30) self.segmentSlider.valueChanged[int].connect(self.changeSegmentValue) # Create the left and right buttons segmentDecrement = QtGui.QPushButton('<', self) segmentDecrement.setToolTip("Back to previous segment") segmentDecrement.clicked[bool].connect(self.segmentDecrement) segmentIncrement = QtGui.QPushButton('>', self) segmentIncrement.setToolTip("Forward to next segment") segmentIncrement.clicked[bool].connect(self.segmentIncrement) # create display mode button self.yDisplay = QtGui.QPushButton("raw") self.yDisplay.setToolTip( "Toggle y axis: raw data or difference=raw-baseline") self.yDisplay.setCheckable(True) self.yDisplay.clicked[bool].connect(self.yDisplayClicked) # create information boxes self.instructionsLabel = QtGui.QLabel() self.instructionsLabel.setText( "ADD peak: a; REMOVE peak: d ZOOM: +/- SCAN l/r ") self.countLabel = QtGui.QLabel() self.countLabel.setText("count label") self.inputLabel = QtGui.QLabel() self.inputLabel.setText("Input File:%s" % self.initialFile) self.goodLabel = QtGui.QLabel() self.goodLabel.setText("Good File:%s" % self.goodFile) # create controls for baseline fitting #self.baseline = QtGui.QPushButton("filter") #self.baseline.setCheckable(True) #self.baseline.clicked[bool].connect(self.baselineClicked) # Create the navigation toolbar, tied to the canvas self.mpl_toolbar = NavigationToolbar(self.canvas, self.main_frame) # Do the layout # segment box segmentBox = QHBoxLayout() segmentBox.addWidget(segmentDecrement) segmentBox.addWidget(self.segmentSlider) segmentBox.addWidget(segmentIncrement) segmentBox.addWidget(self.yDisplay) # baseline box #self.baselineBox = QHBoxLayout() #self.updateBaselineBox() # info box self.infoBox = QVBoxLayout() self.infoBox.addWidget(self.inputLabel) self.infoBox.addWidget(self.goodLabel) self.infoBox.addWidget(self.countLabel) self.infoBox.addWidget(self.instructionsLabel) # entire box vbox = QVBoxLayout() vbox.addLayout(self.infoBox) vbox.addLayout(segmentBox) #vbox.addLayout(self.baselineBox) vbox.addWidget(self.canvas) vbox.addWidget(self.mpl_toolbar) self.main_frame.setLayout(vbox) self.setCentralWidget(self.main_frame) def updateBaselineBox(self): for i in range(self.baselineBox.count()): item = self.baselineBox.itemAt(i) self.baselineBox.removeItem(item) mode = str(self.baseline.text()) self.baseline.setFixedSize(70, 40) self.baselineBox.addWidget(self.baseline) keys = self.fitParams[mode] def load_file(self, fileName): self.wsf = WideSweepFile(fileName) #self.wsf.fitSpline(splineS=1.0, splineK=1) self.wsf.fitFilter(wn=0.01) self.wsf.findPeaks(m=2) self.goodPeakMask = np.zeros(len(self.wsf.x), dtype=np.bool) self.badPeakMask = np.zeros(len(self.wsf.x), dtype=np.bool) self.collMask = np.zeros(len(self.wsf.x), dtype=np.bool) if os.path.isfile( self.baseFile + "-ml-good.txt" ): # update: use machine learning peak loacations if they've been made print 'loading peak location predictions from', self.baseFile + "-ml-good.txt" peaks = np.loadtxt(self.baseFile + "-ml-good.txt") badPeaks = np.loadtxt(self.baseFile + "-ml-bad.txt") peaks = np.array(map(int, peaks), dtype=int) badPeaks = np.atleast_1d(badPeaks) badPeaks = np.array(map(int, badPeaks), dtype=int) self.badPeakMask[badPeaks] = True else: peaks = self.wsf.peaks #remove collisions #freqsAtPeaks = self.wsf.x[peaks] #args = np.argsort(freqsAtPeaks) #peaks=peaks[args] #freqsAtPeaks=freqsAtPeaks[args] #diffs = freqsAtPeaks[1:] - freqsAtPeaks[:-1] #diffs = np.append(diffs, [peaks[-1]]) #peaks=np.delete(peaks, np.where(diffs<2.E-9)) #remove duplicate peaks #freqsAtPeaks=np.delete(freqsAtPeaks, np.where(diffs<2.E-9)) #remove duplicate peaks # #diffs = freqsAtPeaks[1:] - freqsAtPeaks[:-1] #diffs = np.append(diffs, [peaks[-1]]) #colls = peaks[np.where(diffs<=self.coll_thresh)] ''' #coll_thresh = self.wsf.x[0] dist = abs(np.roll(peaks, -1) - peaks) #colls = np.delete(peaks,np.where(dist>=9)) colls=[] diff, coll_thresh = 0, 0 while diff <= 5e-4: diff = self.wsf.x[coll_thresh]-self.wsf.x[0] coll_thresh += 1 print coll_thresh for i in range(len(peaks)): if dist[i]<coll_thresh: if self.wsf.mag[peaks[i+1]] - self.wsf.mag[peaks[i]] > 1.5: colls.append(peaks[i+1]) #print 'for ', self.wsf.x[peaks[i]], 'chosing the one before' else: colls.append(peaks[i]) #print 'for ', self.wsf.x[peaks[i]], 'chosing this one' ''' #print colls #if colls != []: # #colls=np.array(map(int,colls)) # self.collMask[colls] = True # update: so unidentified peaks can be identified as unusable # #peaks = np.delete(peaks,colls) #remove collisions (peaks < 0.5MHz apart = < 9 steps apart) #peaks = np.delete(peaks,np.where(dist<9)) #remove collisions (peaks < 0.5MHz apart = < 9 steps apart) self.goodPeakMask[ peaks] = True #set to false to disable automatic peak finding #self.badPeakMask[colls] = True #set to false to disable automatic peak finding #self.goodPeakMask[colls] = False self.setCountLabel() # self.writeToGoodFile() # self.writeToAllFile() def setCountLabel(self): self.countLabel.setText( "Good peaks = %d, All peaks = %d" % (self.goodPeakMask.sum(), self.goodPeakMask.sum() + self.badPeakMask.sum())) def writeToGoodFile(self): #gf = open(self.goodFile,'wb') ##id = (self.flNum-1)*2000 #resId = self.flNum*10000 #for index in range(len(self.goodPeakMask)): # if self.goodPeakMask[index]: # line = "%8d %12d %16.7f\n"%(resId,index,self.wsf.x[index]) # gf.write(line) # resId += 1 # elif self.badPeakMask[index]: # resId += 1 #gf.close() # wsf.x is the frequency list from the widesweep data file ws_good_inds = np.where(self.goodPeakMask > 0) ws_bad_inds = np.where(self.badPeakMask > 0) freqs = np.append(self.wsf.x[ws_good_inds], self.wsf.x[ws_bad_inds]) sort_inds = np.argsort(freqs) resIds = np.asarray(range(len(freqs))) + self.flNum * 10000 d = np.diff(self.wsf.x[ws_good_inds]) if np.any(d < 0): print "Warning, Some freqs were out of order, check the output file to make sure they were handled correctly" data = np.asarray([resIds[np.where(sort_inds<len(ws_good_inds[0]))], \ ws_good_inds[0][sort_inds[np.where(sort_inds<len(ws_good_inds[0]))]], \ freqs[sort_inds][np.where(sort_inds<len(ws_good_inds[0]))]]).T gf = file( self.goodFile, 'wb' ) #Can open file in append mode this way. But doesn't matter here... np.savetxt(gf, data, fmt="%8d %12d %16.7f") gf.close() def writeToAllFile(self): af = open(self.allFile, 'wb') ##id = (self.flNum-1)*2000 #resId = self.flNum*10000 #print len(np.where(self.goodPeakMask==1)[0]), len(np.where(self.badPeakMask==1)[0]) #for index in range(len(self.goodPeakMask)): # if self.goodPeakMask[index] or self.badPeakMask[index]: # line = "%8d %12d %16.7f\n"%(resId,index,self.wsf.x[index]) # af.write(line) # resId += 1 #af.close() ws_good_inds = np.where(self.goodPeakMask > 0) ws_bad_inds = np.where(self.badPeakMask > 0) freqs = np.append(self.wsf.x[ws_good_inds], self.wsf.x[ws_bad_inds]) sort_inds = np.argsort(freqs) resIds = np.asarray(range(len(freqs))) + self.flNum * 10000 data = np.asarray([ resIds, np.append(ws_good_inds[0], ws_bad_inds[0])[sort_inds], freqs[sort_inds] ]) np.savetxt(af, data.T, fmt='%8d %12d %16.7f') # deal with zooming and plotting one segment def zoom(self, zoom): self.zoomFactor *= zoom self.calcXminXmax() self.plotSegment() def changeSegmentValue(self, value): self.segment = value self.calcXminXmax() self.ylim = self.axes.get_ylim() self.plotSegment() def segmentDecrement(self, value, amount=0.9): wsfDx = self.wsf.x[-1] - self.wsf.x[0] plotDx = self.xMax - self.xMin dsdx = self.segmentMax / wsfDx ds = amount * dsdx * plotDx self.segment = max(0, self.segment - ds) self.segmentSlider.setSliderPosition(self.segment) def segmentIncrement(self, value, amount=0.9): wsfDx = self.wsf.x[-1] - self.wsf.x[0] plotDx = self.xMax - self.xMin dsdx = self.segmentMax / wsfDx ds = amount * dsdx * plotDx self.segment = min(self.segmentMax, self.segment + ds) self.segmentSlider.setSliderPosition(self.segment) def calcXminXmax(self): xMiddle = self.wsf.x[0] + \ (self.segment/self.segmentMax)*(self.wsf.x[-1]-self.wsf.x[0]) dx = self.deltaXDisplay / self.zoomFactor self.xMin = xMiddle - dx / 2.0 self.xMax = xMiddle + dx / 2.0 def seggrouper(self, x): groupedgemask = np.diff(x) <= 0 groupedgemask[0] = 1 #want first group to start at 0 gstarts = np.where(groupedgemask)[0] gstarts[1:] += 1 gends = gstarts[1:] gends = np.append(gends, len(x)) self.slices = map(lambda x: slice(*x), zip(gstarts, gends)) #for s in slices: # yield x[s],y[s] #for start, end in zip(gstarts, gends): # yield x[start:end], y[start:end] def plotSegment(self): ydText = self.yDisplay.text() if self.wsf != None: if ydText == "raw": yPlot = 20 * np.log10(self.wsf.mag) yName = "log-magnitude" else: yPlot = self.wsf.mag - self.wsf.baseline yName = "log(mag-baseline)" stride = self.wsf.data1.shape[0] / self.segmentMax # plot all values and then set xmin and xmax to show this segment self.axes.clear() for i, s in enumerate(self.slices): if self.xMin > self.wsf.x[ s.stop - 1] or self.xMax < self.wsf.x[s.start]: continue x = self.wsf.x[s] y = yPlot[s] use = (x > self.xMin) & (x < self.xMax) self.axes.plot(x[use], y[use], marker='.', markersize=.7, color='C{}'.format(i % 7)) #self.axes.plot(self.wsf.x, yPlot, label=yName) for x in self.wsf.x[self.goodPeakMask]: if x > self.xMin and x < self.xMax: self.axes.axvline(x=x, color='g') self.axes.axvline(x=x + self.coll_thresh, color='g', linestyle='-.', linewidth=0.5) self.axes.axvline(x=x - self.coll_thresh, color='g', linestyle='-.', linewidth=0.5) #for c in self.wsf.x[self.collMask]: # if c > self.xMin and c < self.xMax: # self.axes.axvline(x=c,color='g') for x in self.wsf.x[self.badPeakMask]: if x > self.xMin and x < self.xMax: self.axes.axvline(x=x, color='r') self.axes.axvline(x=x + self.coll_thresh, color='r', linestyle='-.', linewidth=0.5) self.axes.axvline(x=x - self.coll_thresh, color='r', linestyle='-.', linewidth=0.5) self.axes.set_xlim((self.xMin, self.xMax)) #try: self.axes.set_ylim(self.ylim) #except: pass self.axes.set_title("segment=%.1f/%.1f" % (self.segment, self.segmentMax)) #self.axes.legend().get_frame().set_alpha(0.5) plt.tight_layout() self.draw() self.canvas.setFocus() def yDisplayClicked(self, value): if value: self.yDisplay.setText("diff") else: self.yDisplay.setText("raw") self.replot() def baselineClicked(self, value): if value: self.baseline.setText("spline") else: self.baseline.setText("filter") self.updateBaselineBox() def show(self): super(WideAna, self).show() if self.parent == None: self.app.exec_()
class mlClassification(): def __init__(self, inferenceFile=None, nClass = 5, xWidth=40): ''' inputs inferenceFile: the spectrum the user wishes to locate resonators in. If no training pkl files exist this file is used to create training data nClass: Can be either 4 or 5 depending on whether the window width is large enough. If the window width is small, enough examples of collisions are hard to come by in the training data and there wont be enough of this class to match other classes xWidth: The window size to make training data and scan on the inference spectrum. ScalexWidth is an option later which allows the training data with large widths to analyse small windows on the inference spectrum ''' self.baseFile = ('.').join(inferenceFile.split('.')[:-1]) self.goodFile = self.baseFile + '-good.txt' self.mlFile = self.baseFile + '-ml.txt' self.wsf = WideSweepFile(inferenceFile) # use WideSweepFile to get access to the data in inferenceFile self.wsf.fitFilter(wn=0.01) self.wsf.findPeaks() self.nClass = nClass # can be 4 (no collisions class) self.xWidth = xWidth # the width of each frame in number of samples self.trans = self.wsf.mag # transmission self.trainFile = 'train_w%i_c%i.pkl' % (self.xWidth, self.nClass) # a collection of images of xWidth and associated class labels rescaling_factor = 2 # rule of thumb adjustment to amplitudes of each frame so peaks take up more of the frame self.yLims=[min(self.trans), max(self.trans)] self.trainFrac = 0.8 self.testFrac=1 - self.trainFrac self.trans_adjusted= self.trans-min(self.trans) # stretches, normalises and scales the amplitudes to fit on a 0-40 grid self.trans_adjusted = np.round(self.trans_adjusted*rescaling_factor*self.xWidth/max(self.trans_adjusted) ) def makeWindowImage(self, xCenter=25, markers=True, scalexWidth=None, showFrames=False): '''Using a given x coordinate a frame is created at that location of size xWidth x xWidth, and then flattened into a 1d array. Called multiple times in each function. inputs xCenter: center location of frame in wavelength space markers: lines to guide the eye when the frame is shown scalexWidth: typical values: 1/2, 1/4, 1/8 uses interpolation to put data from an xWidth x xWidth grid to a (xWidth/scalexWidth) x (xWidth/scalexWidth) grid. This allows the user to probe the spectrum using a smaller window while utilizing the higher resolution training data showFrames: pops up a window of the frame plotted using matplotlib.plot (used with training) ''' xWidth= self.xWidth # to save pulling from global memory all the time if scalexWidth==None: start = int(xCenter - xWidth/2) end = int(xCenter + xWidth/2) else: start = int(xCenter - xWidth*scalexWidth/2) #can use a smaller window for finer detail and then scale the image later to fit training data end = int(xCenter + xWidth*scalexWidth/2) if end-start != xWidth*scalexWidth: end=end+1 trans = self.trans_adjusted[start:end] trans = map(int, np.array(trans) + (xWidth*4.5/5)-np.median(trans) ) # adjusts the height of the trans data to the median if scalexWidth!=None: x = np.arange(0, xWidth*scalexWidth+1) trans = np.append(trans, trans[-1]) f = interpolate.interp1d(x, trans) xnew = np.arange(0, xWidth*scalexWidth, scalexWidth) trans = f(xnew) image = np.zeros((xWidth, xWidth)) #creates an image of the spectrum for i in range(xWidth-1): if trans[i]>=xWidth: trans[i] = xWidth-1 if trans[i]<0: trans[i] = 0 if trans[i] < trans[i+1]: image[int(trans[i]):int(trans[i+1]),i]=1 else: image[int(trans[i]):int(trans[i-1]),i]=1 if trans[i] == trans[i+1]: image[int(trans[i]),i]=1 try: image[map(int,trans), range(xWidth)]=1 except IndexError: pass if showFrames: fig = plt.figure(frameon=False) fig.add_subplot(111) plt.plot(self.wsf.x[start:end], self.trans[start:end]) if markers: plt.axvline(self.wsf.x[(end-start)/4 + start], ls='dashed') plt.axvline(self.wsf.x[end - (end-start)/4], ls='dashed') self.yLims=[min(self.trans), max(self.trans)] plt.ylim((self.yLims[0],self.yLims[1])) plt.xlim((self.wsf.x[start],self.wsf.x[end])) #plt.axis('off') plt.show() plt.close() image = image.flatten() return image def makeTrainData(self, trainSteps=500): '''creates images of each class with associated labels and saves to pkl file 0: no peak, just flat 1: the center of the peak is 1/3 of the way to the left of center 2: the center of the peak is center of the frame 3: like 1 but to the right 4: the center of the middle peak is in the middle of the frame and there is another peak somewhere to the left of that one s: this image does not belong in the training dataset inputs trainSteps: how many total image does the user want to validate. testFrac adjusts what percentage are for testing outputs trainImages: cube of size- xWidth * xWidth * xCenters*trainFrac trainLabels: 1d array of size- xCenters*trainFrac testImages: cube of size- xWidth * xWidth * xCenters*testFrac testLabels: 1d array of size- xCenters*testFrac ''' self.yLims=[min(self.trans), max(self.trans)] trainImages, trainLabels, testImages, testLabels = [], [], [], [] print self.goodFile if os.path.isfile(self.goodFile): print 'loading peak location data from %s' % self.goodFile peaks = np.loadtxt(self.goodFile)[:,1] else: print 'using WideSweepFile.py to predict where the peaks are' peaks = self.wsf.peaksDict['big'] peakDist = abs(np.roll(peaks, 1) - peaks) colls_thresh = self.xWidth/2 # two peaks in one frame colls = peaks[np.where( peakDist < colls_thresh)[0]]#9 random.shuffle(colls) # to stop multiple peaks appearing in training data of classes 1-4 peaks = peaks[np.where( peakDist >= self.xWidth)[0]] peakDist = abs(np.roll(peaks, 1) - peaks) class_steps = trainSteps/self.nClass if len(colls) < class_steps or self.nClass == 4: print 'no 5th class or not enough collisions detected within the frame width to create one' #colls_class = raw_input('Do you want to create one? [y/n]') #if colls_class=='n': colls=[] self.nClass=4 self.trainFile = 'train_w%i_c%i.pkl' % (self.xWidth, self.nClass) #colls = peaks # same criteria as centrals noise = range(len(self.wsf.x)) # noise locations are randomly selected across array. Hopefully peaks # can be located this way (and labeled as other classes) to remove any # biases from widesweepfile peaks lefts = peaks+self.xWidth/3 centrals = peaks rights = peaks-self.xWidth/3 if os.path.isfile(self.goodFile): noise= peaks[np.where( peakDist > self.xWidth*2)[0]]-self.xWidth # no need for random searches xCenters = np.zeros((class_steps,self.nClass)) xCenters[:,0]=random.sample(noise, class_steps) xCenters[:,1]=random.sample(lefts, class_steps) xCenters[:,2]=random.sample(centrals, class_steps) xCenters[:,3]=random.sample(rights, class_steps) if self.nClass==5: xCenters[:,4]=random.sample(noise, class_steps) for i in range(self.nClass): for j in range(int(self.trainFrac*class_steps) ): image = self.makeWindowImage(xCenter=xCenters[j,i], showFrames=False) trainImages.append(image) one_hot = np.zeros(self.nClass) one_hot[i] = 1 trainLabels.append(one_hot) # A more simple way would be to separate the train and test data after they were read but this did not occur to me #before most of the code was written for i in range(self.nClass): for j in range(int(self.trainFrac*class_steps), int(self.trainFrac*class_steps + self.testFrac*class_steps)) : image = self.makeWindowImage(xCenter=xCenters[j,i], showFrames=False) testImages.append(image) one_hot = np.zeros(self.nClass) one_hot[i] = 1 testLabels.append(one_hot) else: print 'No resonator location file found for this spectrum' append = None if os.path.isfile(self.trainFile): append = raw_input('Do you want to append this training data to previous data [y/n]') if (append == 'y') or (os.path.isfile(self.trainFile)== False): print 'saving files %s & to %s' % (self.trainFile, os.path.dirname(os.path.abspath(self.trainFile)) ) with open(self.trainFile, 'ab') as tf: pickle.dump([trainImages, trainLabels], tf) pickle.dump([testImages, testLabels], tf) return trainImages, trainLabels, testImages, testLabels def mlClass(self): ''' Code adapted from the tensor flow MNIST tutorial 1. Using training images and labels the machine learning class (mlClass) "learns" how to identify peaks. Using similar data the ability of mlClass to identify peaks is tested The training and test matricies are loaded from file (those made earlier if chosen to not be appended to file will not be used) ''' if os.path.isfile(self.trainFile): trainImages, trainLabels, testImages, testLabels = loadPkl(self.trainFile) else: trainImages, trainLabels, testImages, testLabels = self.makeTrainData() print 'Number of training images:', np.shape(trainImages)[0], ' Number of test images:', np.shape(testImages)[0] if np.shape(trainImages)[1]!=self.xWidth**2: print 'Please make new training images of the correct size' exit() self.nClass = np.shape(trainLabels)[1] self.x = tf.placeholder(tf.float32, [None, self.xWidth**2]) # correspond to the images self.W = tf.Variable(tf.zeros([self.xWidth**2, self.nClass])) #the weights used to make predictions on classes self.b = tf.Variable(tf.zeros([self.nClass])) # the biases also used to make class predictions self.y = tf.nn.softmax(tf.matmul(self.x, self.W) + self.b) # class lables predictions made from x,W,b y_ = tf.placeholder(tf.float32, [None, self.nClass]) # true class lables identified by user cross_entropy = -tf.reduce_sum(y_*tf.log(tf.clip_by_value(self.y,1e-10,1.0)) ) # find out how right you are by finding out how wrong you are train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy) # the best result is when the wrongness is minimal init = tf.initialize_all_variables() self.sess = tf.Session() self.sess.run(init) # need to do this everytime you want to access a tf variable (for example the true class labels calculation or plotweights) trainReps = 500 batches = 100 if np.shape(trainLabels)[0]< batches: batches = np.shape(trainLabels)[0]/2 print 'Performing', trainReps, 'training repeats, using batches of', batches for i in range(trainReps): #perform the training step using random batches of images and according labels batch_xs, batch_ys = next_batch(trainImages, trainLabels, batches) self.sess.run(train_step, feed_dict={self.x: batch_xs, y_: batch_ys}) #calculate train_step using feed_dict print 'true class labels: ', self.sess.run(tf.argmax(y_,1), feed_dict={self.x: testImages, y_: testLabels})[:25] #argmax finds the index with the max label value print 'class estimates: ', self.sess.run(tf.argmax(self.y,1), feed_dict={self.x: testImages, y_: testLabels})[:25] #1st 25 printed #print self.sess.run(self.y, feed_dict={self.x: testImages, y_: testLabels})[:100] # print the scores for each class correct_prediction = tf.equal(tf.argmax(self.y,1), tf.argmax(y_,1)) #which ones did it get right? accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) score = self.sess.run(accuracy, feed_dict={self.x: testImages, y_: testLabels}) * 100 print 'Accuracy of model in testing: ', score, '%' if score < 95: print 'Consider making more training images' del trainImages, trainLabels, testImages, testLabels def plotWeights(self): '''creates a 2d map showing the positive and negative weights for each class''' weights = self.sess.run(self.W) weights = np.reshape(weights,(self.xWidth,self.xWidth,self.nClass)) weights = np.flipud(weights) for nc in range(self.nClass): plt.imshow(weights[:,:, nc]) plt.title('class %i' % nc) plt.show() plt.close() def findPeaks(self, inferenceFile, scalexWidth=None, steps =50, start=0, searchWholeSpec=False, useWideAna=True, multi_widths=False): '''The trained machine learning class (mlClass) finds peaks in the inferenceFile spectrum inputs inferenceFile: widesweep data file to be used scalexWidth: allows a smaller/finer window to search spectrum without loss of resolution that comes with smaller window sizes this variable, if set to a number, will act as a multiplication factor for self.xWidth e.g: 0.5 or 0.25 adds to computation time (maybe 4x longer for 0.25) searchWholeSpec: if only a few peaks need to be identified set to False steps: if searchWhileSpec is False, the number of frames (numer of center values in centers) start: if searchWhileSpec is False, the starting point of the frames (first center value in centers) useWideAna: if true once all the peaks have been located these values are fed into WideAna which opens the window where the user can manually check all the peaks have been found and make corrections if neccessary outputs Goodfile: either immediately after the peaks have been located or through WideAna if useWideAna =True mlFile: temporary file read in to WideAna.py containing peak indicies *had some issues running WideAna at the end of this script on ubuntu 14.04 and matplotlib-1.5.1 backend 'Qt4Agg' after running matplotlib.show(). Sometimes I received a segmentation fault ''' try: self.sess except AttributeError: print 'You have to train the model first' exit() self.inferenceFile = inferenceFile self.wsf = WideSweepFile(inferenceFile) self.wsf.fitFilter(wn=0.01) self.wsf.findPeaks() #the center of each frame. xWidth wide, translated xWidth/20 from the previous if scalexWidth != None: xMove = float(self.xWidth*scalexWidth)/40 #1 centers = np.arange(self.xWidth*scalexWidth/2,len(self.wsf.x)-self.xWidth*scalexWidth/2, xMove ) else: xMove = float(self.xWidth)/40 #1 centers = np.arange(self.xWidth/2,len(self.wsf.x)-self.xWidth/2, xMove ) centers = [int(c) for c in centers] print len(self.wsf.x) if searchWholeSpec: steps = len(centers) start=0 span = range(start, steps+start) self.inferenceLabels = np.ones((steps,self.nClass)) print 'Using trained algorithm on images across the inference spectrum' for i,c in enumerate(span): inferenceImage=[] # print how many frames remain to be studied inline sys.stdout.write("\r%d of %i" % (i+1,steps) ) sys.stdout.flush() # each image is formatted into a single element of a list so sess.run can receive a single values dictionary argument # and save memory image = self.makeWindowImage(xCenter=centers[c], scalexWidth=scalexWidth, showFrames=False) # inferenceImage is just reformatted image inferenceImage.append(image) self.inferenceLabels[i,:] = self.sess.run(self.y, feed_dict={self.x: inferenceImage} ) del inferenceImage del image print '\n' stdLabels = np.argmax(self.inferenceLabels, 1) scores = np.zeros((len(stdLabels))) # peaks is an array of positive peak identification locations peaks = np.where(np.logical_or(stdLabels ==2, stdLabels ==4) ) # turn peaks into a list of lists of adjacent peak locations peaks = np.split(peaks[0], np.where(np.diff(peaks[0]) >= 5)[0]+1) #peakLocations takes the middle value of adjacent positive peak identifications in peaks peakLocations=[] if len(peaks[0]) == 0: print 'No peaks found in search range' peakLocations = None else: for p in peaks: if len(p)>3: p = np.array(p) centers = np.array(centers) min_location = np.argmin(self.wsf.mag[centers[p]]) middle = (p[0]+p[len(p)-1]) /2 peakLocations.append(p[min_location]) centers = np.array(centers) peak_dist = abs(np.roll(centers[peakLocations], -1) - centers[peakLocations]) diff, coll_thresh = 0, 0 while diff <= 5e-4: diff = self.wsf.x[coll_thresh]-self.wsf.x[0] coll_thresh += 1 #remove collisions (peaks < 0.5MHz apart). This is also done in WideAna.py if useWideAna: collisions = np.delete(peakLocations,np.where(peak_dist>=coll_thresh)) else: #peakLocations = np.delete(peakLocations,np.where(dist<50)) collisions = [] for i in range(len(peakLocations)): if peak_dist[i]<coll_thresh: if self.wsf.mag[centers[peakLocations[i+1]]] - self.wsf.mag[centers[peakLocations[i]]] > 1.5: collisions.append(centers[peakLocations[i+1]]) else: collisions.append(centers[peakLocations[i]]) peakLocations = np.delete(peakLocations,collisions) if not multi_widths: plt.plot(centers[start:start+steps], self.wsf.mag[centers[start:start+steps]])#- self.wsf.baseline[centers[start:start+steps]]) self.wsf.x[centers...] if peakLocations != []: for pl in peakLocations: plt.axvline(centers[pl+start], color='r') for c in collisions: plt.axvline(centers[c+start], color='g') plt.show() plt.close() print 'Number of resonators located:', len(peakLocations) print 'Number of collisions', len(collisions) # append the peak locations from each window width to this global variable if multi_widths: global mw_peakLocations mw_peakLocations= np.concatenate((mw_peakLocations,centers[map(lambda x: x+start, peakLocations)])) #remove collisions during each successive run to avoid a build up identifications at near collisions mw_peakLocations = np.sort(mw_peakLocations) mw_peakLocations = map(int,mw_peakLocations) peak_dist = abs(np.roll(mw_peakLocations, -1) - mw_peakLocations) collisions = [] for i in range(len(mw_peakLocations)-1): if peak_dist[i]<=3: if self.wsf.mag[mw_peakLocations[i]] <= self.wsf.mag[mw_peakLocations[i+1]]: collisions.append(i) else: collisions.append(i+1) #mw_peakLocations = np.delete(mw_peakLocations,np.where(peak_dist<3)) mw_peakLocations = np.delete(mw_peakLocations,collisions) if useWideAna: # if file exists rename it with the current time if os.path.isfile(self.mlFile): self.mlFile = self.mlFile+time.strftime("-%Y-%m-%d-%H-%M-%S") #shutil.copy(self.mlFile, self.mlFile+time.strftime("-%Y-%m-%d-%H-%M-%S")) mlf = open(self.mlFile,'wb') #mlf machine learning file is temporary if peakLocations != []: if multi_widths: peakLocations =mw_peakLocations peakLocations = np.sort(peakLocations) for pl in peakLocations: line = "%12d\n" % pl# just peak locations mlf.write(line) else: for pl in peakLocations: line = "%12d\n" % centers[pl+start] # just peak locations mlf.write(line) mlf.close() #on ubuntu 14.04 and matplotlib-1.5.1 backend 'Qt4Agg' running matplotlib.show() prior to this causes segmentation fault WideAna.main(initialFile=self.inferenceFile) os.remove(self.mlFile) else: gf = open(self.goodFile,'wb') id = 0 if peakLocations != []: for pl in peakLocations: line = "%8d %12d %16.7f\n"%(id,centers[pl]+start,self.wsf.x[centers[pl]]) gf.write(line) id += 1 gf.close()