def OnPressEnter2(self,event): solution = self.entryVariable2.get() part = solution.split("-") rw_csv.deleteOldGoal(part[1]) item = part[1].split(" ") score = item[len(item)-1] rw_csv.writeScore({part[0]:int(score)}) for i in self.tree.get_children(): self.tree.delete(i) result = rw_csv.readScore() self.tree.insert("" , 0, text="Score", values=(result["Family"],result["Travel"],result["Studying"],result["Friend"],result["Volunteer"])) goal = rw_csv.readGoal() id2 = self.tree.insert("", 1, "Task", text="Task") family_text = "" for x in goal["Family"]: family_text = family_text + x+ "\n" travel_text = "" for x in goal["Travel"]: travel_text = travel_text + x+ "\n" studying_text = "" for x in goal["Studying"]: studying_text = studying_text + x+"\n" friend_text = "" for x in goal["Friend"]: friend_text = friend_text + x+"\n" volunteer_text = "" for x in goal["Volunteer"]: volunteer_text = volunteer_text + x+"\n" self.tree.insert(id2, "end", text="", values=(family_text,travel_text,studying_text,friend_text,volunteer_text)) Analysis.makingBalanceLifeGraph(result["Family"],result["Travel"],result["Studying"],result["Friend"],result["Volunteer"]) img = self.img = ImageTk.PhotoImage(Image.open('BalanceLifeGraph.png')) self.panel = Tkinter.Label(self,image=img) self.panel.pack(side = "left")
def single_processor(filequeue, pointslist, donelist, bgimg = None, basedir = None, thresh=THRESH): while not filequeue.empty(): fname = filequeue.get() frame_num = int(path.splitext(path.split(fname)[-1])[0]) if basedir: pass if len(donelist)%100 == 0: sys.stderr.write("Completed %d\n"%(len(donelist))) sys.stderr.write("Found %d holes\n" % len(pointslist)) try: img1, img2 = Analysis.loadsplit(fname, bgimg = bgimg) pointslist.extend(Analysis.findpairs(img1, img2, thresh=thresh, DEBUG=DEBUG, abs_thresh=True, frame_num = frame_num)) donelist.append(fname) except IOError: pass except KeyboardInterrupt: return
def __init__(self,Pathx, B): path = "/media/iglu/Data/DatasetIglu" i = 0 Analysis.initial() files = open("Output_1_L.txt", 'w') start_time = timeit.default_timer() for x in os.listdir(path): # if i == 0: # i += 1 # continue # else: if Pathx != x: continue for y in os.listdir(path + "/" + x) : if (y != "Objects.txt") and not y.endswith(".zip"): # print path+"/"+x+"/"+y+"/k1" f = open(path + "/" + x + "/" + y + "/k1" + "/List.txt", 'r') f2 = open(path + "/" + x + "/" + y + "/speech.txt", 'r') speak = f2.readline() if speak == 'This...': sp = 1 elif speak == 'That...': sp = 2 else: sp = 0 mn = 0 for line in f: Time = line file1 = next(f).rstrip('\n') file2 = next(f).rstrip('\n') Label = next(f).rstrip('\n') # files.write(RGB.__str__()) # files.write(Depth.__str__()) # files.write(Label+" "+x+" "+file1+"\n") if mn == 0: RGB = cv2.imread(path + "/" + x + "/" + y + "/k1" + "/RGB/" + file1) Depth = cv2.imread(path + "/" + x + "/" + y + "/k1" + "/Depth/" + file2) Analysis.addDepth(Depth) Analysis.addRGB(RGB) Analysis.addSpeech(sp) Analysis.addScene(Label) else: if mn == 2: mn = -1 mn += 1 # print "Enviado " + x + " " + y i += 1 elapsed = timeit.default_timer() - start_time print "Tiempo: " + elapsed.__str__() print "Starting Training" start_time = timeit.default_timer() Analysis.CompleteAnalysis(B, False, True, 0.3, files) files.close() # Analysis.Mostrar() # code you want to evaluate elapsed = timeit.default_timer() - start_time print "Tiempo: " + elapsed.__str__()
def makebackground(filelist, n = 300): import numpy as np from random import randrange as rand filedesc = Analysis.imread(filelist[0]) [r,c] = np.shape(filedesc) mat = np.empty((r,c,n)) for i in range(n): mat[:,:,i] = Analysis.imread(filelist[rand(len(filelist))]) return np.median(mat, axis=2)
def _line_spectrum(data, min, line, dE, width, width_error): # Draw histogram n, bins = Analysis.histogram(data, binsize=binsize) if method in ("cs"): gn, gbins = Analysis.group_bin(n, bins, min=min) else: # No grouping in mle and ls gn, gbins = n, bins ngn = gn/(np.diff(gbins)) ngn_sigma = np.sqrt(gn)/(np.diff(gbins)) cbins = (gbins[1:]+gbins[:-1])/2 if plotting: figure() if width_error is not None: label = 'FWHM$=%.2f\pm %.2f$ eV' % (width, width_error) else: label = 'FWHM$=%.2f$ eV (Fixed)' % width if method == "cs": errorbar(cbins, ngn, yerr=ngn_sigma, xerr=np.diff(gbins)/2, capsize=0, ecolor='k', fmt=None, label=label) else: hist(data, bins=gbins, weights=np.ones(len(data))/binsize, histtype='step', ec='k', label=label) E = np.linspace(bins.min(), bins.max(), 1000) model = Analysis.normalization(ngn, gbins, dE, width, line=line, shift=shift) \ * Analysis.line_model(E, dE, width, line=line, shift=shift, full=True) # Plot theoretical model plot(E, model[0], 'r-') # Plot fine structures for m in model[1:]: plot(E, m, 'b--') xlabel('Energy$\quad$(eV)') ylabel('Normalized Count$\quad$(count/eV)') legend(frameon=False) ymin, ymax = ylim() ylim(ymin, ymax*1.1) tight_layout() savefig("%s-%s.pdf" % (session, line)) if savedat: np.savetxt('%s-%s.dat' % (session, line), np.vstack((cbins, gn)).T, header='Energy (keV), Count', delimiter='\t')
def processor(args): """map-able function that processes a single frame of data Argument: a single tuple composed of the following, in order file_name : string, required pointslist : list, required file_number : ignored, optional background : array, optional donelist : list, optional A list of files that have already been processed status : ?????? """ fname = args[0] frame_num = int(path.splitext(path.split(fname)[-1])[0]) pointslist = args[1] if len(args) > 2: i = args[2] if len(args) > 3: bgimg = args[3] else: bgimg = None if len(args) > 4: donelist = args[4] donelist.append(i) if len(donelist)%1000 == 1: sys.stderr.write("Completed %d\n"%(len(donelist)-1)) sys.stderr.write("Found %d holes\n" % len(pointslist)) if len(donelist)%10000 == 0 and len(pointslist) > 0: xl, yl, varxl, varyl, el, xr, yr, varxr, varyr, er, framenum =\ zip(*list(pointslist)) savemat('tempfile.mat', {'xl':xl, 'yl':yl, 'varxl': varxl, 'xr':xr, 'yr':yr, 'el': el, 'er': er, 'varxr': varxr, 'framenum':framenum}, oned_as = 'row') sys.stderr.write("Saved a snapshot\n") if len(args) > 5: THRESH = args[5] # Finally, load in the image try: img1, img2 = Analysis.loadsplit(fname, bgimg = bgimg) except IOError: print "Failed loading!" return pointslist.extend(Analysis.findpairs(img1, img2, thresh=THRESH, DEBUG=DEBUG, abs_thresh=True, frame_num = frame_num))
def __init__(self, parent, controller): tk.Frame.__init__(self, parent) img = self.img = ImageTk.PhotoImage(Image.open('build_your_career.jpg')) panel = Tkinter.Label(self,image=img) panel.place(x="0",y="0") scrollbar = tk.Scrollbar(self) scrollbar.pack(side="right", fill="y") result = Analysis.infoOLD() text_field = "Quiz: Which Major Is Right For You?\nYour last result: "+result+"\nSource: http://getcollegecredit.com/blog/article/which_major_is_right_for_you\n" f = open('quiz.txt','r') text_field = text_field + str(f.read()) text = tk.Text(self) text.insert("end",text_field) text.pack() text.config(state="disabled",yscrollcommand=scrollbar.set) scrollbar.config(command=text.yview) self.entryVariable = Tkinter.StringVar() self.entry = Tkinter.Entry(self,textvariable=self.entryVariable, width = 50) self.entry.bind("<Return>", self.OnPressEnter) self.entryVariable.set(u"Ex: 1 D,2 C, ...") self.entry.pack() self.labelVariable = Tkinter.StringVar() label = Tkinter.Label(self,textvariable=self.labelVariable,justify="left",wraplength=500,fg="yellow",bg="blue",width = 80,height=10) self.labelVariable.set(u"Hello!") label.pack() button1 = tk.Button(self, text="Back to Home", command=lambda: controller.show_frame(HomePage)) button1.pack()
def __init__(self, data, filename, view, parent): super(DisassemblerView, self).__init__(parent) self.status = "" self.view = view self.data = data for type in ExeFormats: exe = type(data) if exe.valid: self.data = exe self.view.exe = exe break # Create analysis and start it in another thread self.analysis = Analysis(self.data) self.analysis_thread = threading.Thread(None, self.analysis_thread_proc) self.analysis_thread.daemon = True self.analysis_thread.start() # Start disassembly view at the entry point of the binary if hasattr(self.data, "entry"): self.function = self.data.entry() else: self.function = None self.update_id = None self.ready = False self.desired_pos = None self.highlight_token = None self.cur_instr = None self.scroll_mode = False self.blocks = {} self.show_il = False self.simulation = None # Create timer to automatically refresh view when it needs to be updated self.updateTimer = QTimer() self.updateTimer.setInterval(100) self.updateTimer.setSingleShot(False) self.updateTimer.timeout.connect(self.updateTimerEvent) self.updateTimer.start() self.initFont() # Initialize scroll bars self.width = 0 self.height = 0 self.setHorizontalScrollBarPolicy(Qt.ScrollBarAsNeeded) self.setVerticalScrollBarPolicy(Qt.ScrollBarAsNeeded) self.horizontalScrollBar().setSingleStep(self.charWidth) self.verticalScrollBar().setSingleStep(self.charHeight) areaSize = self.viewport().size() self.adjustSize(areaSize.width(), areaSize.height()) # Setup navigation self.view.register_navigate("disassembler", self, self.navigate) self.view.register_navigate("make_proc", self, self.make_proc) self.search_regex = None self.last_search_type = FindDialog.SEARCH_HEX
def analyzeParsedSmali(classes,sharedobjs): # Begin Analysis log.info("Analysis Started") dependencies = {} dependencies["all"],dependencies["internal"],dependencies["external"],dependencies["unknown"] = getDependencies(classes) results = Analysis.runAnalysis(classes,dependencies,sharedobjs) log.saveLibrariesToPickle(dependencies["all"]) return results
def get_total_SSRO_events(pqf, RO_start, RO_length, marker_chan, chan_rnd_0, chan_rnd_1, sync_time_lim, VERBOSE = True): """ Returns SSRO data for all marked events. Colums are: Sync Nymber | number of photons | RND number indicator | RND number | Sync Time RND number | Sync Times photon 1-24 | """ _a = get_attributes(pqf) # Gets the number of blocks in the data num_blocks = Analysis.get_num_blocks(pqf) if VERBOSE or (num_blocks > 1): print 'The total number of blocks is:', num_blocks # Initializes arrays to save the PQ-data PQ_sync_number = np.empty((0,), dtype = np.uint32) PQ_special = np.empty((0,), dtype = np.uint32) PQ_sync_time = np.empty((0,), dtype = np.uint64) PQ_time = np.empty((0,), dtype = np.uint64) PQ_channel = np.empty((0,), dtype = np.uint32) # Initializes an array to save the SSRO data total_SSRO_events = np.empty((0,30), dtype = np.uint64) # Loops over every block for i in range(num_blocks): # Get the SSRO events and PQ data for these sync numbers gc.collect() _events, _PQ_sync_number, _PQ_special, _PQ_sync_time, _PQ_time, _PQ_channel = \ get_SSRO_events(pqf, marker_chan, RO_start, RO_length, chan_rnd_0, chan_rnd_1, sync_time_lim = sync_time_lim, index = i+1, VERBOSE = VERBOSE) # Concatenates all PQ data PQ_sync_number = np.hstack((PQ_sync_number,_PQ_sync_number)) PQ_special = np.hstack((PQ_special, _PQ_special)) PQ_sync_time = np.hstack((PQ_sync_time, _PQ_sync_time)) PQ_time = np.hstack((PQ_time, _PQ_time)) PQ_channel = np.hstack((PQ_channel, _PQ_channel)) # Stacks all SSRO data total_SSRO_events = np.vstack((total_SSRO_events, _events)) if VERBOSE>2: print print 'Found {} valid marked SSRO events in block'.format(int(len(_events))), i+1 print '====================================' print if VERBOSE: print print 'Found {} valid marked SSRO events in all blocks'.format(int(len(total_SSRO_events))) print '====================================' print return total_SSRO_events, _a, PQ_sync_number, PQ_special, PQ_sync_time, PQ_time, PQ_channel
def __init__(self, P, lstOfGenerationsNumbers): outputPath = os.path.dirname(os.path.realpath(__file__)) + "/" f = open(outputPath+"Stats.txt","w") f.write(" ====================================================== \n") f.write("|| ||\n") f.write("|| Statistical Properties of this simulation run ||\n") f.write("|| ||\n") f.write(" ====================================================== \n\n\n") f.write("Number of Agents : {:>5}\n".format(P.numberAgents)) f.write("Maximum size of social groups : {:>5}\n".format(P.maxSocSize)) f.write("Maximum number of Generations : {:>5}\n".format(P.generations)) f.write("Number of individual CultEvo runs: {:>5}\n\n\n\n".format(P.numberOfSimulationRuns)) f.write("Values:\n") f.write("=======\n\n") f.write("Averages and measures of central location\n") f.write(".........................................\n\n") f.write("mean : {:>6.2f}\n".format(stat.mean(lstOfGenerationsNumbers))) f.write("median : {:>6.2f}\n".format(stat.median(lstOfGenerationsNumbers))) f.write("mean_low : {:>6.2f}\n".format(stat.median_low(lstOfGenerationsNumbers))) f.write("mean_high : {:>6.2f}\n".format(stat.median_high(lstOfGenerationsNumbers))) f.write("mean_grouped: {:>6.2f}\n".format(stat.median_grouped(lstOfGenerationsNumbers))) try: f.write("mode : {:>6.2f}\n\n\n\n".format(stat.mode(lstOfGenerationsNumbers) )) except : f.write("mode : two equal values found\n") f.write("Measures of spread\n") f.write("..................\n\n") f.write("Population standard deviation of data: {0:>6.2f}\n".format(stat.pstdev(lstOfGenerationsNumbers))) f.write("Population variance of data : {0:>6.2f}\n".format(stat.pvariance(lstOfGenerationsNumbers))) f.write("Sample standard deviation of data : {0:>6.2f}\n".format(stat.stdev(lstOfGenerationsNumbers))) f.write("Sample variance of data : {0:>6.2f}\n".format(stat.variance(lstOfGenerationsNumbers))) f.close()
def OnPressEnter(self,event): solution = self.entryVariable.get() data_point = rw_csv.readStudyHrs() x = [] y = [] for point in data_point: x.append(int(point[0])) y.append(int(point[1])) (anal,predict_value) = Analysis.regression(x,y,int(solution)) self.text.insert("end",anal+"You should study for "+str(predict_value)+" hours for next exam!!!") self.text.pack() self.entry.focus_set() self.entry.selection_range(0, Tkinter.END)
def __init__(self): def nothing(x): pass self.node_name = "cv_bridge_demo" rospy.init_node(self.node_name) self.BoW = BoW.BoW("/home/iglu/catkin_ws/src/RGBDHand/src/bof.pkl") Analysis.initial() # What we do during shutdown rospy.on_shutdown(self.cleanup) # Create the OpenCV display window for the RGB image self.cv_window_name = self.node_name cv.NamedWindow(self.cv_window_name, cv.CV_WINDOW_NORMAL) cv.MoveWindow(self.cv_window_name, 25, 75) # And one for the depth image cv.NamedWindow("Depth Image", cv.CV_WINDOW_NORMAL) cv.MoveWindow("Depth Image", 25, 350) # And one for the depth image cv.NamedWindow("Histogram", cv.CV_WINDOW_NORMAL) cv.MoveWindow("Histogram", 480, 350) self.bridge = CvBridge() self.Im_p = Image_Process.Image_Process() # Subscribe to the camera image and depth topics and set # the appropriate callbacks self.RGB_sub = message_filters.Subscriber('/camera/rgb/image_raw', Image) self.Depth_sub = message_filters.Subscriber('/camera/depth/image_raw', Image) # rospy.Subscriber('Scene',String,callbackS) self.ts = message_filters.ApproximateTimeSynchronizer([self.RGB_sub, self.Depth_sub], 1,1) # self.image_sub = rospy.Subscriber("/camera/rgb/image_rect_color", Image, self.image_callback) # self.depth_sub = rospy.Subscriber("/camera/depth/image_raw", Image, self.depth_callback) self.ts.registerCallback(self.image_callback) self.depth = np.zeros((480,680)) rospy.loginfo("Waiting for image topics...")
def _line_fit(data, min, line): # Fit (dE, width), (dE_error, width_error), e = Analysis.fit(data, binsize=binsize, min=min, line=line, shift=shift, method=method) if method == "cs": chi_squared, dof = e if method in ("mle", "ls"): print "%s: %.2f +/- %.2f eV @ Ec%+.2f eV" \ % (line, width, width_error, dE) elif method == "cs": print "%s: %.2f +/- %.2f eV @ Ec%+.2f eV (Red. chi^2 = %.1f/%d = %.2f)" \ % (line, width, width_error, dE, chi_squared, dof, chi_squared/dof) return dE, width, width_error
def __init__(self,B): path = "/media/iglu/Data/Dataset/DatasetIglu/Dataset_cleaned_v4" i = 0 Analysis.initial() files = open("Output_1_L.txt", 'w') start_time = timeit.default_timer() for x in os.listdir(path): # if i == 0: # i += 1 # continue # else: f = open(path+"/"+x+"/List.txt",'r') for line in f: Time = line file1 = next(f).rstrip('\n') file2 = next(f).rstrip('\n') Label = next(f).rstrip('\n') RGB = cv2.imread(path+"/"+x+"/RGB/"+file1) Depth = cv2.imread(path+"/"+x+"/Depth/"+file2) # files.write(RGB.__str__()) # files.write(Depth.__str__()) # files.write(Label+" "+x+" "+file1+"\n") Analysis.addDepth(Depth) Analysis.addRGB(RGB) Analysis.addScene(Label) i += 1 print "Enviado "+x # if i > 150: # break elapsed = timeit.default_timer() - start_time print "Tiempo: "+elapsed.__str__() print "Starting Training" start_time = timeit.default_timer() Analysis.CompleteAnalysis(B,True,0.3,files) files.close() #Analysis.Mostrar() # code you want to evaluate elapsed = timeit.default_timer() - start_time print "Tiempo: "+elapsed.__str__()
def dumbClustering(wordList, fName = 'clusters.txt'): """ Clusters by starting each comment as it's own cluster, and seeing which other clusters have a lot of words in common with it No stemming, all words are included It's pretty bad, really. """ #make clusters allClusters = [] for c in wordList: tempCluster = Cluster() freqTuple = Analysis.wordFrequency([c]) if len(freqTuple) == 0: continue freqDict = {} for w, f in freqTuple: freqDict[w] = f tempCluster.addComment(c, freqDict) allClusters.append(tempCluster) #play with these values start = 1.0 mid = .6 end = .3 step = -1.0/15 #combine clusters dumbClusterStep(start, mid, step, allClusters) singleClusters = Cluster() for j in range(len(allClusters)-1, -1, -1): if allClusters[j].numComments() <= 5: singleClusters.addCluster(allClusters[j]) del allClusters[j] allClusters.append(singleClusters) dumbClusterStep(mid, end, step, allClusters) f = open(fName, 'w') for c in allClusters: f.write(str(c)+'\n') f.close()
def basic(constants): """ Runs a basic experiment Parameters: -``constants``: Config settings for the EA from the config files specified at run time """ try: random.seed(constants["seed"]) except KeyError: pass results = [] for run in range(constants["runs"]): constants["runNumber"] = run evaluation = Util.moduleClasses(Fitness)[constants["problem"]](constants).eval results.append(oneRun(constants, evaluation)) data = [] for i in results: data.append(i[-1][1]) return Analysis.meanstd(data), results
def get_total_SSRO_events_quick(pqf, RO_start, RO_length, marker_chan, sync_time_lim, VERBOSE = True): """ Returns quick SSRO data for all marked events. Sync Nymber | number of photons | Sync time first photon """ columns = "Sync_Number,Number of photons,Sync_Time_photon_1" _a = {'Columns': columns} # Gets the number of blocks for the data num_blocks = Analysis.get_num_blocks(pqf) if VERBOSE: print 'The total number of blocks is:', num_blocks # Initializes the array to save all SSRO events total_SSRO_events = np.empty((0,3)) # Loop over all blocks for i in range(num_blocks): _events = get_SSRO_events_quick(pqf, marker_chan, RO_start, RO_length, sync_time_lim = sync_time_lim, index = i+1) # Stacks all events for several blocks total_SSRO_events = np.vstack((total_SSRO_events, _events)) if VERBOSE: print print 'Found {} valid marked SSRO events in block'.format(int(len(_events))), i+1 print '====================================' print if VERBOSE: print print 'Found {} valid marked SSRO events in all blocks'.format(int(len(total_SSRO_events))) print '====================================' print return total_SSRO_events, _a
def handleCmdButton4(self): #check to see if data has been loaded test = self.checkData() if( test == False): return self.size = None self.color = None #check to see if there is any data in the set after filters applied if( self.dataObject.filtersize() == 0): tkMessageBox.showwarning("No data!","Remove Filters from the data or load a different data set!") return print "handling command button 4" # run pca on all of the numerical data self.Analysis = Analysis(name = "PCA", data = self.dataObject.getData_num()) #dictioanry that will store the headers for the pca analsysis plots self.pcadataHeader = {} # this will run the dialog for pca inputBox = PCADialog(parent = self.root, pcaDataHeader = self.pcadataHeader, displayClass = self, Analysis = self.Analysis) if( self.pcadataHeader['check'] == True): self.xAxisLabel.set( self.pcadataHeader.get('x') ) self.yAxisLabel.set( self.pcadataHeader.get('y') ) z = self.pcadataHeader.get('z') #z is optional if( z == None ): z = "" self.zAxisLabel.set(z) self.buildLabels() #graphs the pca points self.buildPCA() if( self.pcadataHeader.get('saveData') ): filename = self.pcadataHeader.get('filename') filename = filename + ".csv" self.Analysis.writeData(filename = filename)
limit_inner = None if len(sys.argv) == 5: if sys.argv[4] != 'None': limit_inner = float(sys.argv[4]) fname = basedir + '/' + tag + '_%i_XCO_P8_limit_inner.hdf5' % limit_inner fix_xco = False if len(sys.argv) == 6: if sys.argv[5] == '0': fix_xco == True # Load the analysis A = Analysis.Analysis( tag='P7REP_CLEAN_V15_calore', fglpath='/pfs/carlson/gll_psc_v14.fit', templateDir='/home/carlson/pfs/Extended_archive_v15/Templates', basepath='/pfs/carlson/GCE_sys/') # A.GenPointSourceTemplate(pscmap=(A.basepath + '/PSC_all_sky_3fgl.npy')) # A.BinPhotons(outfile='binned_photons_all_sky.npy') #A.GenSquareMask(l_range=[-180.,180], b_range=[-40.,40.], plane_mask=1.) A.BinPhotons(infile='binned_photons_all_sky.npy') # Load 2FGL A.AddPointSourceTemplate(fixNorm=True, pscmap=('PSC_3FGL_with_ext.npy')) A.CalculatePixelWeights(diffuse_model='fermi_diffuse_' + A.tag + '.npy', psc_model='PSC_3FGL_with_ext.npy', alpha_psc=5., f_psc=0.2) A.AddIsotropicTemplate( fixNorm=False, fixSpectrum=True ) # External chi^2 used to fix normalization within uncertainties
Drawing.DrawLetter(Folder, Filename) # if there are three arguments set the file and folder name if len(sys.argv) == 3: if sys.argv[1] == "alphabet": DrawingsPerLetter = int(sys.argv[2]) for Item in listdir("./Letters/"): for i in range(0, DrawingsPerLetter): Folder = Item SaveAsUnusedName() elif sys.argv[1] == "analyze": # actually do the drawing Folder = sys.argv[2] SaveAsUnusedName() # get the analytics data AnalysisData = Analysis.RankingsFromFile(Folder, Filename, "FullCharacterRecognition") for i in range(0, 3): print AnalysisData[i]["Value"], "letter", AnalysisData[i]["Letter"] if AnalysisData[0]["Letter"] == Folder: os.remove("Letters/" + Folder + "/" + Filename) print "removing reduntant sample." else: Filename = sys.argv[2] Folder = sys.argv[1] # actually do the drawing Drawing.DrawLetter(Folder, Filename)
import os import Analysis Count = 0 Path = "E:/Data/PageSources" Data = os.listdir(Path) for i in Data: InData = os.listdir(Path + "/" + i) for j in InData: Count += 1 print(Count) InPath = Path + "/" + i + "/" + j Analysis.FunctionGamda(InPath, i)
entree_co_labels = pickEntree_df.axes[0].tolist() entree_company = entree_co_labels[0] matchFoundUpdate(entree_company, preference) matchFound = 1 if matchFound == 0: # all solutions have failed entree_dict['Entree Seating:'].append("failed") dessert_dict['Dessert Seating:'].append("failed") # Now, Combine all of the dictionaries created into a data frame results = Merge(firstName_dict, lastName_dict, eID_dict, year_dict, email_dict, entree_dict, dessert_dict, major_dict) results_df = pd.DataFrame(results) # Create a data frame of only the successful assignments assignments_df = results_df[~( (results_df['Entree Seating:'].str.contains('failed')))] # Create a Pandas Excel writer using XlsxWriter as the engine. writer = pd.ExcelWriter('Results_2018.xlsx', engine='xlsxwriter') # Convert the assignments dataframe to an XlsxWriter Excel object. results_df.to_excel(writer, sheet_name='All Results') assignments_df.to_excel(writer, sheet_name='Successful Assignments') # Close the Pandas Excel writer and output the Excel file. writer.save() import Analysis Analysis.analyze_preferences(preference_dessert_dict, preference_entree_dict) Analysis.analyze_major_dist(assignments_df, results_df)
import Analysis if __name__ == '__main__': totalReceiver = int(input('Enter number of receivers : ')) wTable = WalshTableGenerator.getWalshTable(totalReceiver) output_file = "to.txt" receiverThreadPool = [] receiverList = [] for index in range(0, totalReceiver): new_receiver = Receiver.Receiver(index, output_file, wTable[index]) receiverList.append(new_receiver) new_receiver_thread = threading.Thread( target=new_receiver.startReceive, args=()) receiverThreadPool.append(new_receiver_thread) new_receiver_thread.start() for index in range(0, totalReceiver): receiverThreadPool[index].join() pktCount = 0 totalTime = 0 for index in range(0, totalReceiver): pktCount += receiverList[index].report.pktCount totalTime += receiverList[index].report.totalTime totalTime /= totalReceiver finalReport = Analysis.Report(pktCount, totalTime) finalReport.storeReport(totalReceiver)
class DisassemblerView(QAbstractScrollArea): statusUpdated = Signal(QWidget, name="statusUpdated") def __init__(self, data, filename, view, parent): super(DisassemblerView, self).__init__(parent) self.status = "" self.view = view self.data = data for type in ExeFormats: exe = type(data) if exe.valid: self.data = exe self.view.exe = exe break # Create analysis and start it in another thread self.analysis = Analysis(self.data) self.analysis_thread = threading.Thread(None, self.analysis_thread_proc) self.analysis_thread.daemon = True self.analysis_thread.start() # Start disassembly view at the entry point of the binary if hasattr(self.data, "entry"): self.function = self.data.entry() else: self.function = None self.update_id = None self.ready = False self.desired_pos = None self.highlight_token = None self.cur_instr = None self.scroll_mode = False self.blocks = {} self.show_il = False self.simulation = None # Create timer to automatically refresh view when it needs to be updated self.updateTimer = QTimer() self.updateTimer.setInterval(100) self.updateTimer.setSingleShot(False) self.updateTimer.timeout.connect(self.updateTimerEvent) self.updateTimer.start() self.initFont() # Initialize scroll bars self.width = 0 self.height = 0 self.setHorizontalScrollBarPolicy(Qt.ScrollBarAsNeeded) self.setVerticalScrollBarPolicy(Qt.ScrollBarAsNeeded) self.horizontalScrollBar().setSingleStep(self.charWidth) self.verticalScrollBar().setSingleStep(self.charHeight) areaSize = self.viewport().size() self.adjustSize(areaSize.width(), areaSize.height()) # Setup navigation self.view.register_navigate("disassembler", self, self.navigate) self.view.register_navigate("make_proc", self, self.make_proc) self.search_regex = None self.last_search_type = FindDialog.SEARCH_HEX def initFont(self): # Get font and compute character sizes self.font = getMonospaceFont() self.baseline = int(QFontMetricsF(self.font).ascent()) self.charWidth = QFontMetricsF(self.font).width('X') self.charHeight = int(QFontMetricsF( self.font).height()) + getExtraFontSpacing() self.charOffset = getFontVerticalOffset() def adjustSize(self, width, height): # Recompute size information self.renderWidth = self.width self.renderHeight = self.height self.renderXOfs = 0 self.renderYOfs = 0 if self.renderWidth < width: self.renderXOfs = int((width - self.renderWidth) / 2) self.renderWidth = width if self.renderHeight < height: self.renderYOfs = int((height - self.renderHeight) / 2) self.renderHeight = height # Update scroll bar information self.horizontalScrollBar().setPageStep(width) self.horizontalScrollBar().setRange(0, self.renderWidth - width) self.verticalScrollBar().setPageStep(height) self.verticalScrollBar().setRange(0, self.renderHeight - height) def resizeEvent(self, event): # Window was resized, adjust scroll bar self.adjustSize(event.size().width(), event.size().height()) def get_cursor_pos(self): if self.cur_instr is None: return self.function return self.cur_instr def set_cursor_pos(self, addr): if not self.view.navigate("disassembler", addr): self.view_in_hex_editor(addr) def get_selection_range(self): return (self.get_cursor_pos(), self.get_cursor_pos()) def set_selection_range(self, begin, end): self.set_cursor_pos(begin) def write(self, data): pos = self.get_cursor_pos() if pos is None: return False return self.data.write(pos, data) == len(data) def copy_address(self): clipboard = QApplication.clipboard() clipboard.clear() mime = QMimeData() mime.setText("0x%x" % self.get_cursor_pos()) clipboard.setMimeData(mime) def analysis_thread_proc(self): self.analysis.analyze() def closeRequest(self): # Stop analysis when closing tab self.analysis.stop() return True def paintEvent(self, event): # Initialize painter p = QPainter(self.viewport()) p.setFont(self.font) xofs = self.horizontalScrollBar().value() yofs = self.verticalScrollBar().value() if not self.ready: # Analysis for the current function is not yet complete, paint loading screen gradient = QLinearGradient( QPointF(0, 0), QPointF(self.viewport().size().width(), self.viewport().size().height())) gradient.setColorAt(0, QColor(232, 232, 232)) gradient.setColorAt(1, QColor(192, 192, 192)) p.setPen(QColor(0, 0, 0, 0)) p.setBrush(QBrush(gradient)) p.drawRect(0, 0, self.viewport().size().width(), self.viewport().size().height()) if self.function is None: text = "No function selected" else: text = "Loading..." p.setPen(Qt.black) p.drawText((self.viewport().size().width() / 2) - ((len(text) * self.charWidth) / 2), (self.viewport().size().height() / 2) + self.charOffset + self.baseline - (self.charHeight / 2), text) return # Render background gradient = QLinearGradient( QPointF(-xofs, -yofs), QPointF(self.renderWidth - xofs, self.renderHeight - yofs)) gradient.setColorAt(0, QColor(232, 232, 232)) gradient.setColorAt(1, QColor(192, 192, 192)) p.setPen(QColor(0, 0, 0, 0)) p.setBrush(QBrush(gradient)) p.drawRect(0, 0, self.viewport().size().width(), self.viewport().size().height()) p.translate(self.renderXOfs - xofs, self.renderYOfs - yofs) # Render each node for block in self.blocks.values(): # Render shadow p.setPen(QColor(0, 0, 0, 0)) p.setBrush(QColor(0, 0, 0, 128)) p.drawRect(block.x + self.charWidth + 4, block.y + self.charWidth + 4, block.width - (4 + 2 * self.charWidth), block.height - (4 + 2 * self.charWidth)) # Render node background gradient = QLinearGradient( QPointF(0, block.y + self.charWidth), QPointF(0, block.y + block.height - self.charWidth)) gradient.setColorAt(0, QColor(255, 255, 252)) gradient.setColorAt(1, QColor(255, 255, 232)) p.setPen(Qt.black) p.setBrush(QBrush(gradient)) p.drawRect(block.x + self.charWidth, block.y + self.charWidth, block.width - (4 + 2 * self.charWidth), block.height - (4 + 2 * self.charWidth)) if self.cur_instr != None: y = block.y + (2 * self.charWidth) + ( len(block.block.header_text.lines) * self.charHeight) for instr in block.block.instrs: if instr.addr == self.cur_instr: p.setPen(QColor(0, 0, 0, 0)) p.setBrush(QColor(255, 255, 128, 128)) p.drawRect(block.x + self.charWidth + 3, y, block.width - (10 + 2 * self.charWidth), len(instr.text.lines) * self.charHeight) y += len(instr.text.lines) * self.charHeight if self.highlight_token: # Render highlighted tokens x = block.x + (2 * self.charWidth) y = block.y + (2 * self.charWidth) for line in block.block.header_text.tokens: for token in line: if token[2:] == self.highlight_token: p.setPen(QColor(0, 0, 0, 0)) p.setBrush(QColor(192, 0, 0, 64)) p.drawRect(x + token[0] * self.charWidth, y, token[1] * self.charWidth, self.charHeight) y += self.charHeight for instr in block.block.instrs: for line in instr.text.tokens: for token in line: if token[2:] == self.highlight_token: p.setPen(QColor(0, 0, 0, 0)) p.setBrush(QColor(192, 0, 0, 64)) p.drawRect(x + token[0] * self.charWidth, y, token[1] * self.charWidth, self.charHeight) y += self.charHeight # Render node text x = block.x + (2 * self.charWidth) y = block.y + (2 * self.charWidth) for line in block.block.header_text.lines: partx = x for part in line: p.setPen(part[1]) p.drawText(partx, y + self.charOffset + self.baseline, part[0]) partx += len(part[0]) * self.charWidth y += self.charHeight for instr in block.block.instrs: for line in instr.text.lines: partx = x for part in line: p.setPen(part[1]) p.drawText(partx, y + self.charOffset + self.baseline, part[0]) partx += len(part[0]) * self.charWidth y += self.charHeight # Render edges for edge in block.edges: p.setPen(edge.color) p.setBrush(edge.color) p.drawPolyline(edge.polyline) p.drawConvexPolygon(edge.arrow) def isMouseEventInBlock(self, event): # Convert coordinates to system used in blocks xofs = self.horizontalScrollBar().value() yofs = self.verticalScrollBar().value() x = event.x() + xofs - self.renderXOfs y = event.y() + yofs - self.renderYOfs # Check each block for hits for block in self.blocks.values(): # Compute coordinate relative to text area in block blockx = x - (block.x + (2 * self.charWidth)) blocky = y - (block.y + (2 * self.charWidth)) # Check to see if click is within bounds of block if (blockx < 0) or (blockx > (block.width - 4 * self.charWidth)): continue if (blocky < 0) or (blocky > (block.height - 4 * self.charWidth)): continue return True return False def getInstrForMouseEvent(self, event): # Convert coordinates to system used in blocks xofs = self.horizontalScrollBar().value() yofs = self.verticalScrollBar().value() x = event.x() + xofs - self.renderXOfs y = event.y() + yofs - self.renderYOfs # Check each block for hits for block in self.blocks.values(): # Compute coordinate relative to text area in block blockx = x - (block.x + (2 * self.charWidth)) blocky = y - (block.y + (2 * self.charWidth)) # Check to see if click is within bounds of block if (blockx < 0) or (blockx > (block.width - 4 * self.charWidth)): continue if (blocky < 0) or (blocky > (block.height - 4 * self.charWidth)): continue # Compute row within text row = int(blocky / self.charHeight) # Determine instruction for this row cur_row = len(block.block.header_text.lines) if row < cur_row: return block.block.entry for instr in block.block.instrs: if row < cur_row + len(instr.text.lines): return instr.addr cur_row += len(instr.text.lines) return None def getTokenForMouseEvent(self, event): # Convert coordinates to system used in blocks xofs = self.horizontalScrollBar().value() yofs = self.verticalScrollBar().value() x = event.x() + xofs - self.renderXOfs y = event.y() + yofs - self.renderYOfs # Check each block for hits for block in self.blocks.values(): # Compute coordinate relative to text area in block blockx = x - (block.x + (2 * self.charWidth)) blocky = y - (block.y + (2 * self.charWidth)) # Check to see if click is within bounds of block if (blockx < 0) or (blockx > (block.width - 4 * self.charWidth)): continue if (blocky < 0) or (blocky > (block.height - 4 * self.charWidth)): continue # Compute row and column within text col = int(blockx / self.charWidth) row = int(blocky / self.charHeight) # Check tokens to see if one was clicked cur_row = 0 for line in block.block.header_text.tokens: if cur_row == row: for token in line: if (col >= token[0]) and (col < (token[0] + token[1])): # Clicked on a token return token cur_row += 1 for instr in block.block.instrs: for line in instr.text.tokens: if cur_row == row: for token in line: if (col >= token[0]) and (col < (token[0] + token[1])): # Clicked on a token return token cur_row += 1 return None def find_instr(self, addr): for block in self.blocks.values(): for instr in block.block.instrs: if instr.addr == addr: return instr return None def nop_out(self, addr): instr = self.find_instr(addr) if instr != None: self.view.begin_undo() instr.patch_to_nop(self.data) self.view.commit_undo() def always_branch(self, addr): instr = self.find_instr(addr) if instr != None: self.view.begin_undo() instr.patch_to_always_branch(self.data) self.view.commit_undo() def invert_branch(self, addr): instr = self.find_instr(addr) if instr != None: self.view.begin_undo() instr.patch_to_invert_branch(self.data) self.view.commit_undo() def skip_and_return_zero(self, addr): instr = self.find_instr(addr) if instr != None: self.view.begin_undo() instr.patch_to_zero_return(self.data) self.view.commit_undo() def skip_and_return_value(self, addr): instr = self.find_instr(addr) if instr != None: value, ok = QInputDialog.getText(self, "Skip and Return Value", "Return value:", QLineEdit.Normal) if ok: try: value = int(value, 0) except: QMessageBox.critical(self, "Error", "Expected numerical address") return self.view.begin_undo() instr.patch_to_fixed_return_value(self.data, value) self.view.commit_undo() def view_in_hex_editor(self, addr): if not self.view.navigate("exe", addr): self.view.navigate("hex", addr) def show_address(self): if "address" in self.analysis.options: addr = False else: addr = True self.analysis.set_address_view(addr) def context_menu(self, addr): popup = QMenu() view_in_hex = popup.addAction("View in &hex editor") view_in_hex.triggered.connect(lambda: self.view_in_hex_editor(addr)) view_in_hex.setShortcut(QKeySequence(Qt.Key_H)) popup.addAction("Copy address", self.copy_address) enter_name_action = popup.addAction("Re&name symbol", self.enter_name) enter_name_action.setShortcut(QKeySequence(Qt.Key_N)) undefine_name_action = popup.addAction("&Undefine symbol", self.undefine_name) undefine_name_action.setShortcut(QKeySequence(Qt.Key_U)) show_address_action = popup.addAction("Show &address", self.show_address) show_address_action.setCheckable(True) show_address_action.setChecked("address" in self.analysis.options) popup.addSeparator() patch = popup.addMenu("&Patch") patch.addAction("Convert to NOP").triggered.connect( lambda: self.nop_out(addr)) instr = self.find_instr(addr) if instr: if instr.is_patch_branch_allowed(): patch.addAction("Never branch").triggered.connect( lambda: self.nop_out(addr)) patch.addAction("Always branch").triggered.connect( lambda: self.always_branch(addr)) patch.addAction("Invert branch").triggered.connect( lambda: self.invert_branch(addr)) if instr.is_patch_to_zero_return_allowed(): patch.addAction("Skip and return zero").triggered.connect( lambda: self.skip_and_return_zero(addr)) if instr.is_patch_to_fixed_return_value_allowed(): patch.addAction("Skip and return value...").triggered.connect( lambda: self.skip_and_return_value(addr)) popup.exec_(QCursor.pos()) def mousePressEvent(self, event): if (event.button() != Qt.LeftButton) and (event.button() != Qt.RightButton): return if not self.isMouseEventInBlock(event): # Click outside any block, enter scrolling mode self.scroll_base_x = event.x() self.scroll_base_y = event.y() self.scroll_mode = True self.viewport().grabMouse() return # Check for click on a token and highlight it token = self.getTokenForMouseEvent(event) if token: self.highlight_token = token[2:] else: self.highlight_token = None # Update current instruction instr = self.getInstrForMouseEvent(event) if instr != None: self.cur_instr = instr else: self.cur_instr = None self.viewport().update() if (instr != None) and (event.button() == Qt.RightButton): self.context_menu(instr) def mouseMoveEvent(self, event): if self.scroll_mode: x_delta = self.scroll_base_x - event.x() y_delta = self.scroll_base_y - event.y() self.scroll_base_x = event.x() self.scroll_base_y = event.y() self.horizontalScrollBar().setValue( self.horizontalScrollBar().value() + x_delta) self.verticalScrollBar().setValue( self.verticalScrollBar().value() + y_delta) def mouseReleaseEvent(self, event): if event.button() != Qt.LeftButton: return if self.scroll_mode: self.scroll_mode = False self.viewport().releaseMouse() def mouseDoubleClickEvent(self, event): token = self.getTokenForMouseEvent(event) if token and (token[2] == "ptr"): self.analysis.lock.acquire() if not self.analysis.functions.has_key(token[3]): # Not a function or not analyzed, go to address in hex editor addr = token[3] self.analysis.lock.release() self.view_in_hex_editor(addr) else: self.view.add_history_entry() self.function = token[3] self.ready = False self.desired_pos = None self.cur_instr = None self.highlight_token = None self.viewport().update() self.analysis.lock.release() def go_to_address(self): addr_str, ok = QInputDialog.getText(self, "Go To Address", "Address:", QLineEdit.Normal) if ok: try: addr = int(addr_str, 16) if (addr < self.data.start()) or (addr > self.data.end()): if hasattr(self.data, "symbols_by_name") and ( addr_str in self.data.symbols_by_name): addr = self.data.symbols_by_name[addr_str] else: QMessageBox.critical(self, "Error", "Address out of range") return except: if hasattr(self.data, "symbols_by_name") and ( addr_str in self.data.symbols_by_name): addr = self.data.symbols_by_name[addr_str] elif (addr_str[0] == '@') and hasattr( self.data, "symbols_by_name") and (addr_str[1:] in self.data.symbols_by_name): addr = self.data.symbols_by_name[addr_str[1:]] else: QMessageBox.critical(self, "Error", "Invalid address or symbol") return # Try navigating within disassembly, if it isn't within a function then # navigate to the hex editor if not self.view.navigate("disassembler", addr): self.view_in_hex_editor(addr) def enter_name(self): # A symbol must be selected if (self.highlight_token == None) or (self.highlight_token[0] != "ptr"): QMessageBox.critical(self, "Error", "No symbol selected.") return addr = self.highlight_token[1] name = self.highlight_token[2] # Ask for new name new_name, ok = QInputDialog.getText(self, "Rename Symbol", "Symbol name:", QLineEdit.Normal, name) if ok: self.analysis.create_symbol(addr, new_name) def undefine_name(self): # A symbol must be selected if (self.highlight_token == None) or (self.highlight_token[0] != "ptr"): QMessageBox.critical(self, "Error", "No symbol selected.") return addr = self.highlight_token[1] name = self.highlight_token[2] # Ask for new name self.analysis.undefine_symbol(addr, name) def navigate_for_find(self, addr): func, instr = self.analysis.find_instr(addr, True) if func != None: self.navigate(addr) else: self.make_proc(addr) self.cur_instr = addr self.desired_pos = None def perform_find(self, dlg): self.search_regex = dlg.search_regex() if self.cur_instr != None: self.search_start = self.cur_instr else: if self.function is None: return self.search_start = self.function found_loc = self.data.find(self.search_regex, self.search_start) if found_loc != -1: self.view.add_history_entry() self.navigate_for_find(found_loc) self.search_pos = found_loc + 1 return found_loc = self.data.find(self.search_regex, self.data.start()) if (found_loc != -1) and (found_loc < self.search_start): self.view.add_history_entry() self.navigate_for_find(found_loc) self.search_pos = found_loc + 1 return QMessageBox.information(self, "Not Found", "Search string not found.") def find(self): dlg = FindDialog(self.last_search_type, self) if dlg.exec_() == QDialog.Accepted: self.last_search_type = dlg.search_type() self.perform_find(dlg) def find_next(self): if self.search_regex == None: QMessageBox.critical(self, "Error", "No active search") return found_loc = self.data.find(self.search_regex, self.search_pos) if self.search_pos >= self.search_start: if found_loc != -1: self.view.add_history_entry() self.navigate_for_find(found_loc) self.search_pos = found_loc + 1 return self.search_pos = 0 else: if (found_loc != -1) and (found_loc < self.search_start): self.view.add_history_entry() self.navigate_for_find(found_loc) self.search_pos = found_loc + 1 return QMessageBox.information(self, "End of Search", "No additional matches found.") self.search_pos = self.search_start return found_loc = self.data.find(self.search_regex, self.search_pos) if found_loc < self.search_start: self.view.add_history_entry() self.navigate_for_find(found_loc) self.search_pos = found_loc + 1 return QMessageBox.information(self, "End of Search", "No additional matches found.") self.search_pos = self.search_start def keyPressEvent(self, event): if event.key() == Qt.Key_H: if self.cur_instr != None: self.view_in_hex_editor(self.cur_instr) else: if self.function is not None: self.view_in_hex_editor(self.function) elif event.key() == Qt.Key_G: self.go_to_address() elif event.key() == Qt.Key_N: self.enter_name() elif event.key() == Qt.Key_U: self.undefine_name() elif event.key() == Qt.Key_Slash: dlg = FindDialog(FindDialog.SEARCH_REGEX, self) if dlg.exec_() == QDialog.Accepted: self.perform_find(dlg) else: super(DisassemblerView, self).keyPressEvent(event) def prepareGraphNode(self, block): # Compute size of node in pixels width = 0 height = 0 for line in block.block.header_text.lines: chars = 0 for part in line: chars += len(part[0]) if chars > width: width = chars height += 1 for instr in block.block.instrs: for line in instr.text.lines: chars = 0 for part in line: chars += len(part[0]) if chars > width: width = chars height += 1 block.width = (width + 4) * self.charWidth + 4 block.height = (height * self.charHeight) + (4 * self.charWidth) + 4 def adjustGraphLayout(self, block, col, row): block.col += col block.row += row for edge in block.new_exits: self.adjustGraphLayout(self.blocks[edge], col, row) def computeGraphLayout(self, block): # Compute child node layouts and arrange them horizontally col = 0 row_count = 1 for edge in block.new_exits: self.computeGraphLayout(self.blocks[edge]) self.adjustGraphLayout(self.blocks[edge], col, 1) col += self.blocks[edge].col_count if (self.blocks[edge].row_count + 1) > row_count: row_count = self.blocks[edge].row_count + 1 block.row = 0 if col >= 2: # Place this node centered over the child nodes block.col = (col - 2) / 2 block.col_count = col else: # No child nodes, set single node's width (nodes are 2 columns wide to allow # centering over a branch) block.col = 0 block.col_count = 2 block.row_count = row_count def isEdgeMarked(self, edges, row, col, index): if index >= len(edges[row][col]): return False return edges[row][col][index] def markEdge(self, edges, row, col, index): while len(edges[row][col]) <= index: edges[row][col] += [False] edges[row][col][index] = True def findHorizEdgeIndex(self, edges, row, min_col, max_col): # Find a valid index i = 0 while True: valid = True for col in range(min_col, max_col + 1): if self.isEdgeMarked(edges, row, col, i): valid = False break if valid: break i += 1 # Mark chosen index as used for col in range(min_col, max_col + 1): self.markEdge(edges, row, col, i) return i def findVertEdgeIndex(self, edges, col, min_row, max_row): # Find a valid index i = 0 while True: valid = True for row in range(min_row, max_row + 1): if self.isEdgeMarked(edges, row, col, i): valid = False break if valid: break i += 1 # Mark chosen index as used for row in range(min_row, max_row + 1): self.markEdge(edges, row, col, i) return i def routeEdge(self, horiz_edges, vert_edges, edge_valid, start, end, color): edge = DisassemblerEdge(color, end) # Find edge index for initial outgoing line i = 0 while True: if not self.isEdgeMarked(vert_edges, start.row + 1, start.col + 1, i): break i += 1 self.markEdge(vert_edges, start.row + 1, start.col + 1, i) edge.addPoint(start.row + 1, start.col + 1) edge.start_index = i horiz = False # Find valid column for moving vertically to the target node if end.row < (start.row + 1): min_row = end.row max_row = start.row + 1 else: min_row = start.row + 1 max_row = end.row col = start.col + 1 if min_row != max_row: ofs = 0 while True: col = start.col + 1 - ofs if col >= 0: valid = True for row in range(min_row, max_row + 1): if not edge_valid[row][col]: valid = False break if valid: break col = start.col + 1 + ofs if col < len(edge_valid[min_row]): valid = True for row in range(min_row, max_row + 1): if not edge_valid[row][col]: valid = False break if valid: break ofs += 1 if col != (start.col + 1): # Not in same column, need to generate a line for moving to the correct column if col < (start.col + 1): min_col = col max_col = start.col + 1 else: min_col = start.col + 1 max_col = col index = self.findHorizEdgeIndex(horiz_edges, start.row + 1, min_col, max_col) edge.addPoint(start.row + 1, col, index) horiz = True if end.row != (start.row + 1): # Not in same row, need to generate a line for moving to the correct row index = self.findVertEdgeIndex(vert_edges, col, min_row, max_row) edge.addPoint(end.row, col, index) horiz = False if col != (end.col + 1): # Not in ending column, need to generate a line for moving to the correct column if col < (end.col + 1): min_col = col max_col = end.col + 1 else: min_col = end.col + 1 max_col = col index = self.findHorizEdgeIndex(horiz_edges, end.row, min_col, max_col) edge.addPoint(end.row, end.col + 1, index) horiz = True # If last line was horizontal, choose the ending edge index for the incoming edge if horiz: index = self.findVertEdgeIndex(vert_edges, end.col + 1, end.row, end.row) edge.points[len(edge.points) - 1][2] = index return edge def renderFunction(self, func): # Create render nodes self.blocks = {} for block in func.blocks.values(): self.blocks[block.entry] = DisassemblerBlock(block) self.prepareGraphNode(self.blocks[block.entry]) # Populate incoming lists for block in self.blocks.values(): for edge in block.block.exits: self.blocks[edge].incoming += [block.block.entry] # Construct acyclic graph where each node is used as an edge exactly once block = func.blocks[func.entry] visited = [func.entry] queue = [self.blocks[func.entry]] changed = True while changed: changed = False # First pick nodes that have single entry points while len(queue) > 0: block = queue.pop() for edge in block.block.exits: if edge in visited: continue # If node has no more unseen incoming edges, add it to the graph layout now if len(self.blocks[edge].incoming) == 1: self.blocks[edge].incoming.remove(block.block.entry) block.new_exits += [edge] queue += [self.blocks[edge]] visited += [edge] changed = True # No more nodes satisfy constraints, pick a node to continue constructing the graph best = None for block in self.blocks.values(): if not block.block.entry in visited: continue for edge in block.block.exits: if edge in visited: continue if (best == None) or (len( self.blocks[edge].incoming) < best_edges) or ( (len(self.blocks[edge].incoming) == best_edges) and (edge < best)): best = edge best_edges = len(self.blocks[edge].incoming) best_parent = block if best != None: self.blocks[best].incoming.remove(best_parent.block.entry) best_parent.new_exits += [best] visited += [best] changed = True # Compute graph layout from bottom up self.computeGraphLayout(self.blocks[func.entry]) # Prepare edge routing horiz_edges = [None] * (self.blocks[func.entry].row_count + 1) vert_edges = [None] * (self.blocks[func.entry].row_count + 1) edge_valid = [None] * (self.blocks[func.entry].row_count + 1) for row in range(0, self.blocks[func.entry].row_count + 1): horiz_edges[row] = [None] * (self.blocks[func.entry].col_count + 1) vert_edges[row] = [None] * (self.blocks[func.entry].col_count + 1) edge_valid[row] = [True] * (self.blocks[func.entry].col_count + 1) for col in range(0, self.blocks[func.entry].col_count + 1): horiz_edges[row][col] = [] vert_edges[row][col] = [] for block in self.blocks.values(): edge_valid[block.row][block.col + 1] = False # Perform edge routing for block in self.blocks.values(): start = block for edge in block.block.exits: end = self.blocks[edge] color = Qt.black if edge == block.block.true_path: color = QColor(0, 144, 0) elif edge == block.block.false_path: color = QColor(144, 0, 0) start.edges += [ self.routeEdge(horiz_edges, vert_edges, edge_valid, start, end, color) ] # Compute edge counts for each row and column col_edge_count = [0] * (self.blocks[func.entry].col_count + 1) row_edge_count = [0] * (self.blocks[func.entry].row_count + 1) for row in range(0, self.blocks[func.entry].row_count + 1): for col in range(0, self.blocks[func.entry].col_count + 1): if len(horiz_edges[row][col]) > row_edge_count[row]: row_edge_count[row] = len(horiz_edges[row][col]) if len(vert_edges[row][col]) > col_edge_count[col]: col_edge_count[col] = len(vert_edges[row][col]) # Compute row and column sizes col_width = [0] * (self.blocks[func.entry].col_count + 1) row_height = [0] * (self.blocks[func.entry].row_count + 1) for block in self.blocks.values(): if (int(block.width / 2)) > col_width[block.col]: col_width[block.col] = int(block.width / 2) if (int(block.width / 2)) > col_width[block.col + 1]: col_width[block.col + 1] = int(block.width / 2) if int(block.height) > row_height[block.row]: row_height[block.row] = int(block.height) # Compute row and column positions col_x = [0] * self.blocks[func.entry].col_count row_y = [0] * self.blocks[func.entry].row_count self.col_edge_x = [0] * (self.blocks[func.entry].col_count + 1) self.row_edge_y = [0] * (self.blocks[func.entry].row_count + 1) x = 16 for i in range(0, self.blocks[func.entry].col_count): self.col_edge_x[i] = x x += 8 * col_edge_count[i] col_x[i] = x x += col_width[i] y = 16 for i in range(0, self.blocks[func.entry].row_count): self.row_edge_y[i] = y y += 8 * row_edge_count[i] row_y[i] = y y += row_height[i] self.col_edge_x[self.blocks[func.entry].col_count] = x self.row_edge_y[self.blocks[func.entry].row_count] = y self.width = x + 16 + ( 8 * col_edge_count[self.blocks[func.entry].col_count]) self.height = y + 16 + ( 8 * row_edge_count[self.blocks[func.entry].row_count]) # Compute node positions for block in self.blocks.values(): block.x = int((col_x[block.col] + col_width[block.col] + 4 * col_edge_count[block.col + 1]) - (block.width / 2)) if (block.x + block.width) > (col_x[block.col] + col_width[block.col] + col_width[block.col + 1] + 8 * col_edge_count[block.col + 1]): block.x = int((col_x[block.col] + col_width[block.col] + col_width[block.col + 1] + 8 * col_edge_count[block.col + 1]) - block.width) block.y = row_y[block.row] # Precompute coordinates for edges for block in self.blocks.values(): for edge in block.edges: start = edge.points[0] start_row = start[0] start_col = start[1] last_index = edge.start_index last_pt = QPoint( self.col_edge_x[start_col] + (8 * last_index) + 4, block.y + block.height + 4 - (2 * self.charWidth)) pts = [last_pt] for i in range(0, len(edge.points)): end = edge.points[i] end_row = end[0] end_col = end[1] last_index = end[2] if start_col == end_col: new_pt = QPoint( last_pt.x(), self.row_edge_y[end_row] + (8 * last_index) + 4) else: new_pt = QPoint( self.col_edge_x[end_col] + (8 * last_index) + 4, last_pt.y()) pts += [new_pt] last_pt = new_pt start_col = end_col new_pt = QPoint(last_pt.x(), edge.dest.y + self.charWidth - 1) pts += [new_pt] edge.polyline = pts pts = [ QPoint(new_pt.x() - 3, new_pt.y() - 6), QPoint(new_pt.x() + 3, new_pt.y() - 6), new_pt ] edge.arrow = pts # Adjust scroll bars for new size areaSize = self.viewport().size() self.adjustSize(areaSize.width(), areaSize.height()) if self.desired_pos: # There was a position saved, navigate to it self.horizontalScrollBar().setValue(self.desired_pos[0]) self.verticalScrollBar().setValue(self.desired_pos[1]) elif self.cur_instr != None: self.show_cur_instr() else: # Ensure start node is visible start_x = self.blocks[func.entry].x + self.renderXOfs + int( self.blocks[func.entry].width / 2) self.horizontalScrollBar().setValue(start_x - int(areaSize.width() / 2)) self.verticalScrollBar().setValue(0) self.update_id = func.update_id self.ready = True self.viewport().update(0, 0, areaSize.width(), areaSize.height()) def updateTimerEvent(self): status = self.analysis.status if status != self.status: self.status = status self.statusUpdated.emit(self) if self.function is None: return if self.ready: # Check for updated code self.analysis.lock.acquire() if self.update_id != self.analysis.functions[ self.function].update_id: self.renderFunction(self.analysis.functions[self.function]) self.analysis.lock.release() return # View not up to date, check to see if active function is ready self.analysis.lock.acquire() if self.analysis.functions.has_key(self.function): if self.analysis.functions[self.function].ready: # Active function now ready, generate graph self.renderFunction(self.analysis.functions[self.function]) self.analysis.lock.release() def show_cur_instr(self): for block in self.blocks.values(): row = len(block.block.header_text.lines) for instr in block.block.instrs: if self.cur_instr == instr.addr: x = block.x + int(block.width / 2) y = block.y + (2 * self.charWidth) + int( (row + 0.5) * self.charHeight) self.horizontalScrollBar().setValue( x + self.renderXOfs - int(self.horizontalScrollBar().pageStep() / 2)) self.verticalScrollBar().setValue( y + self.renderYOfs - int(self.verticalScrollBar().pageStep() / 2)) return row += len(instr.text.lines) def navigate(self, addr): # Check to see if address is within current function for block in self.blocks.values(): row = len(block.block.header_text.lines) for instr in block.block.instrs: if (addr >= instr.addr) and (addr < (instr.addr + len(instr.opcode))): self.cur_instr = instr.addr self.show_cur_instr() self.viewport().update() return True row += len(instr.text.lines) # Check other functions for this address func, instr = self.analysis.find_instr(addr) if func != None: self.function = func self.cur_instr = instr self.highlight_token = None self.ready = False self.desired_pos = None self.viewport().update() return True return False def make_proc(self, addr): # Create a procedure at the requested address if one does not already exist if self.data.architecture() is None: # Architecture not defined yet, ask the user and set it now arch_dlg = ArchitectureDialog(self) if arch_dlg.exec_() == QDialog.Rejected: return False self.data.default_arch = arch_dlg.result self.analysis.lock.acquire() if addr not in self.analysis.functions: self.analysis.queue.append(addr) self.analysis.lock.release() self.function = addr self.cur_instr = None self.highlight_token = None self.ready = False self.desired_pos = None self.viewport().update() return True def navigate_to_history_entry(self, entry): self.function = entry.function self.ready = False self.desired_pos = [entry.scroll_x, entry.scroll_y] self.cur_instr = entry.cur_instr self.highlight_token = entry.highlight_token self.viewport().update() def get_history_entry(self): return DisassemblerHistoryEntry(self) def fontChanged(self): self.initFont() if self.ready: # Rerender function to update layout self.analysis.lock.acquire() self.renderFunction(self.analysis.functions[self.function]) self.analysis.lock.release() def getPriority(data, ext): if Analysis.isPreferredForFile(data): return 80 return 0 getPriority = staticmethod(getPriority) def getViewName(): return "Disassembler" getViewName = staticmethod(getViewName) def getShortViewName(): return "Disassembler" getShortViewName = staticmethod(getShortViewName) def handlesNavigationType(name): return (name == "disassembler") or (name == "make_proc") handlesNavigationType = staticmethod(handlesNavigationType)
def main(): serial_number='' path_root='../Tests' offsets='' test_date='' date = str(datetime.datetime.now()).split(' ')[0] for arg in sys.argv: if arg.split('=')[0]=='serial_number': serial_number=arg.split('=')[1] if arg.split('=')[0]=='test_date': test_date=arg.split('=')[1] path=path_root+'/'+serial_number offsets='' with open(path+'/data/offsets.csv','r') as csvfile: count=0 plots = csv.reader(csvfile, delimiter=',') for row in plots: if count > 0: offsets+=', '+row[0] else: offsets+=row[0] count+=1 Analysis.report='y' Analysis.data='bias' Analysis.path = path Analysis.trial_id = '1' Analysis.ave_time='1' Analysis.num_points=15 Analysis.show_plot='n' Analysis.save_plot='y' bias_out=Analysis.analyze() Analysis.report='y' Analysis.data='dac' Analysis.path = path Analysis.trial_id = '1' Analysis.ave_time='1' Analysis.num_points=15 Analysis.show_plot='n' Analysis.save_plot='y' dac_out=Analysis.analyze() Analysis.report='y' Analysis.data='current' Analysis.path = path Analysis.trial_id = '1' Analysis.ave_time='1' Analysis.num_points=15 Analysis.show_plot='n' Analysis.save_plot='y' current_out_one=Analysis.analyze() Analysis.report='y' Analysis.data='current' Analysis.path = path Analysis.trial_id = '1' Analysis.ave_time='100' Analysis.num_points=15 Analysis.show_plot='n' Analysis.save_plot='y' current_out_hund=Analysis.analyze() document=R''' %\documentclass{revtex4-1} \documentclass{article}% %\usepackage{amsmath}% %\usepackage{amsfonts}% \usepackage{amssymb}% \usepackage{graphicx,float} \usepackage{amsmath} \usepackage{geometry} \usepackage{booktabs} \usepackage{float} \usepackage{makecell} \geometry{letterpaper} \begin{document} \title{NSLS2\_EM Serial \#'''+serial_number+r''' Test Report} \author{Kon Aoki} %\\Colorado College} \date{'''+date+r'''} \maketitle \section{Summary} The NSLS2\_EM electrometer serial \#'''+serial_number+r''' was tested for its accuracy of readouts. %PROCEDURE \section{Procedure} \begin{enumerate} \item Record ADC offsets \item Grounding \begin{enumerate} \item Confirm chassis ground to mains ground \item Confirm outer conductor of coax connector grounded to chassis \end{enumerate} \item Bias (Test date: '''+test_date+r''') \begin{enumerate} \item Set bias via EPICS \label{itm:1s} \item Measure and record bias output \item Repeat measurement 3 times \label{itm:1l} \item Repeat procedure \ref{itm:1s}-\ref{itm:1l} for biases -10 to 10V in 1V increments \end{enumerate} \item DAC Output (Test date: '''+test_date+r''') \begin{enumerate} \item Set DAC via EPICS \label{itm:2s} \item Measure and record DAC output \label{itm:2s2} \item Repeat measurement 3 times \label{itm:2l} \item Repeat procedure \ref{itm:2s}-\ref{itm:2l} for outputs -10 to 10V in 1V increments \label{itm:l2} \item Repeat all steps for each channel \end{enumerate} \item Current measurement at 1ms averaging time (Test date: '''+test_date+r''') \begin{enumerate} \item Set values per read to 50 \item Set averaging time to 1ms \item Set number of points to collect to 100 \item Set ADC offset \item Set range \label{itm:3s} \item Set input current \label{itm:3s2} \item Wait 5.1 seconds until data is ready (averaging time $\times$ number of points $+$ $5$s buffer) \item Record current measurement \label{itm:3l} \item Repeat procedure \ref{itm:3s2}-\ref{itm:3l} for 15 inputs between 0 and 20\% above the range setting \item Repeat procedure \ref{itm:3s}-\ref{itm:3l} for ranges 1$\mu$A to 50mA \item Repeat all steps for each channel \end{enumerate} \item Current measurement at 100ms averaging time (Test date: '''+test_date+r''') \begin{enumerate} \item Set values per read to 50 \item Set averaging time to 100ms \item Set number of points to collect to 100 \item Set ADC offset \item Set range \label{itm:4s} \item Set input current \label{itm:4s2} \item Wait 15 seconds until data is ready (averaging time $\times$ number of points $+$ $5$s buffer) \item Record current measurement \label{itm:4l} \item Repeat procedure \ref{itm:4s2}-\ref{itm:4l} for 15 inputs between 0 and 20\% above the range setting \item Repeat procedure \ref{itm:4s}-\ref{itm:4l} for ranges 1$\mu$A to 50mA \item Repeat all steps for each channel \end{enumerate} \end{enumerate} %RESULTS \section{Results} ADC Offsets were set at '''+offsets+r'''. \\ Confirmed chassis ground to mains ground\\ Confirmed outer conductor of coax connector grounded to chassis. %Bias \subsection{Bias Voltage} \begin{figure}[H] \centering \includegraphics[width=1\textwidth]{plot/bias.png} \caption{Bias Voltage} \label{fig_bias} \end{figure} \begin{table}[H] \centering \begin{tabular}{|l|l|l|} \hline Slope & Offset (V) & Median STD (V)\\ \hline '''+bias_out+r''' \end{tabular} \end{table} %DAC \subsection{DAC Output} \begin{figure}[H] \centering \includegraphics[width=1\textwidth]{plot/dac.png} \caption{DAC Output} \label{fig_dac} \end{figure} \begin{table}[H] \centering \begin{tabular}{|l|l|l|l|} \hline Channel & Slope & Offset (V) & Median STD (V)\\ \hline '''+dac_out+r''' \end{tabular} \end{table} %CURRENT MEASUREMENT 1ms \subsection{Current Measurement 1ms Averaging Time} *Note that measurements taken from input values above the range setting was ignored for the model for ranges 1$\mu$A to 1mA. For the 50mA range, input values above 40mA (80\%) was ignored for the model. \begin{figure}[H] \centering \includegraphics[width=0.55\textwidth]{plot/1ms_range1.png} \caption{Range 1$\mu$A} \label{fig_adc} \end{figure} \begin{figure}[H] \centering \includegraphics[width=0.55\textwidth]{plot/1ms_range10.png} \caption{Range 10$\mu$A} \label{fig_adc2} \end{figure} \begin{figure}[H] \centering \includegraphics[width=0.55\textwidth]{plot/1ms_range100.png} \caption{Range 100$\mu$A} \label{fig_adc3} \end{figure} \begin{figure}[H] \centering \includegraphics[width=0.55\textwidth]{plot/1ms_range1000.png} \caption{Range 1mA} \label{fig_adc4} \end{figure} \begin{figure}[H] \centering \includegraphics[width=0.55\textwidth]{plot/1ms_range50000.png} \caption{Range 50mA} \label{fig_adc5} \end{figure} \begin{table}[H] \centering \begin{tabular}{|l|l|l|l|l|} \hline Channel & Range ($\mu$A) & Slope & Offset ($\mu$A) & Median STD ($\mu$A)\\ \Xhline{3\arrayrulewidth} '''+current_out_one+r''' \end{tabular} \end{table} %CURRENT MEASUREMENT 100ms \subsection{Current Measurement 100ms Averaging Time} *Note that measurements taken from input values above the range setting was ignored for the model for ranges 1$\mu$A to 1mA. For the 50mA range, input values above 40mA (80\%) was ignored for the model. \begin{figure}[H] \centering \includegraphics[width=0.55\textwidth]{plot/100ms_range1.png} \caption{Range 1$\mu$A} \label{fig_adc} \end{figure} \begin{figure}[H] \centering \includegraphics[width=0.55\textwidth]{plot/100ms_range10.png} \caption{Range 10$\mu$A} \label{fig_adc2} \end{figure} \begin{figure}[H] \centering \includegraphics[width=0.55\textwidth]{plot/100ms_range100.png} \caption{Range 100$\mu$A} \label{fig_adc3} \end{figure} \begin{figure}[H] \centering \includegraphics[width=0.55\textwidth]{plot/100ms_range1000.png} \caption{Range 1mA} \label{fig_adc4} \end{figure} \begin{figure}[H] \centering \includegraphics[width=0.55\textwidth]{plot/100ms_range50000.png} \caption{Range 50mA} \label{fig_adc5} \end{figure} \begin{table}[H] \centering \begin{tabular}{|l|l|l|l|l|} \hline Channel & Range ($\mu$A) & Slope & Offset ($\mu$A) & Median STD ($\mu$A)\\ \Xhline{3\arrayrulewidth} '''+current_out_hund+r''' \end{tabular} \end{table} \end{document} ''' report_path=path+"/Unit"+serial_number+"_report.tex" print("Report generated at "+report_path) f = open(report_path,"w") f.write(document) f.close()
def main(experiment, experimentName, n=1): print("Started Runs") start: float = timeit.default_timer() finishedRuns = [] numberOfIterations = list(range(n)) byTarget = {"makespan": {}, "avgFlowTime": {}, "maximumLateness": {}} for target in byTarget: for sf in experiment["schedulers"]: byTarget[target][sf] = [] if Path(experimentName + ".csv").is_file(): with open(experimentName + ".csv", newline="") as csvfile: reader = csv.reader(csvfile, delimiter=";") for row in reader: byTarget[row[0]][row[1]].append((float(row[2]), float(row[3]))) #print(", ".join(row)) product = itertools.product(*(list(experiment.values()))[:-2]) for conf in product: #itertools.product(numberOfJobs,numberOfNodes,seqR,largeR,timespan,minSeq,maxSeq,minPar,maxPar): for i in numberOfIterations: jobs: List[Simulation.Job] = Generator.generate(*conf) for sf in experiment["schedulers"]: sys: Simulation.System = Simulation.System( jobs.copy(), conf[1], schedulerConverter[sf]) finished: List[Simulation.Job] = sys.run() analysis = Analysis.standardAnalysis(finished) for target in byTarget: byTarget[target][sf].append( (conf[xValueConverter[experiment["x-axis"][0]]], analysis[target])) #print (byTarget) #sortieren nach x #dbCopy = {**bySchedulers} #save to file: with open(experimentName + ".csv", "w", newline="") as csvfile: writer = csv.writer(csvfile, delimiter=";") for target in byTarget: for sf in byTarget[target]: for valuePair in byTarget[target][sf]: writer.writerow([target] + [sf] + list(valuePair)) fig, axs = plt.subplots(3) i = 0 for targetFunktion in ["makespan", "avgFlowTime", "maximumLateness"]: #setup xLabel = experiment["x-axis"][0] yLabel = targetFunktion for sf in byTarget[targetFunktion]: xs = [] ys = [] pairs = sorted(byTarget[targetFunktion][sf], key=lambda x: x[0]) grouped = itertools.groupby(pairs, lambda x: x[0]) for key, group in grouped: vals = list(group) xs.append(key) ys.append(sum(map(lambda x: x[1], vals)) / len(vals)) axs[i].plot(xs, ys, label=sf) axs[i].set_ylabel(targetFunktion) axs[i].set_ylim(ymin=0) axs[i].legend(bbox_to_anchor=(1.0, 1), loc="upper left", fontsize="xx-small") i += 1 plt.xlabel(experiment["x-axis"][0]) plt.show()
def threadCrown(filepath): global io rtpSkel=-1 crownT=OrderedDict() imgL=[] stemCorrection=bool(int(options[8][1])) print io.getHomePath() oldHome=io.getHomePath() os.chdir(io.getHomePath()) io.setHomePath('./Crown/') f=io.scanDir() for (counter,i) in enumerate(f): io.setFileName(os.path.basename(i)) io.setidIdx(imgID) print 'processing Crown file: '+i xScale=allPara[counter][7] yScale=allPara[counter][8] analysis=Analysis.Analysis(io,(xScale+yScale)/2) rtp=RootTipPaths.RootTipPaths(io) try: img=scipy.misc.imread(i,flatten=True) except: print 'Image not readable' img=-1 if len(img)>0: seg=Segmentation.Segmentation(img,io) imgL=seg.label() print 'compute root profile' currT=time.time() if ifAnyKeyIsTrue(['AVG_DENSITY','WIDTH_MED','WIDTH_MAX','DIA_STM_SIMPLE','D10','D20','D30','D40','D50','D60','D70','D80','D90','DS10','DS20','DS30','DS40','DS50','DS60','DS70','DS80','DS90','AREA','ANG_TOP','ANG_BTM']): crownT['AVG_DENSITY'],crownT['WIDTH_MED'],crownT['WIDTH_MAX'],crownT['D10'],crownT['D20'],crownT['D30'],crownT['D40'],crownT['D50'],crownT['D60'],crownT['D70'],crownT['D80'],crownT['D90'],crownT['DS10'],crownT['DS20'],crownT['DS30'],crownT['DS40'],crownT['DS50'],crownT['DS60'],crownT['DS70'],crownT['DS80'],crownT['DS90'],crownT['AREA'],crownT['DIA_STM_SIMPLE'],crownT['ANG_TOP'],crownT['ANG_BTM']=analysis.getWidthOverHeight(imgL,xScale,yScale) print 'Mask traits computed '+str(time.time()-currT)+'s' if ifAnyKeyIsTrue(['DIA_STM','TD_MED','TD_AVG','STA_RANGE','STA_DOM_I','STA_DOM_II','STA_25_I','STA_25_II','STA_50_I','STA_50_II','STA_75_I','STA_75_II','STA_90_I','STA_90_II','RTA_DOM_I','RTA_DOM_II','STA_MIN','STA_MAX','STA_MED','RTA_RANGE','RTA_MIN','RTA_MAX','RTA_MED','NR_RTP_SEG_I','NR_RTP_SEG_II','ADVT_COUNT','BASAL_COUNT','ADVT_ANG','BASAL_ANG','HYP_DIA','TAP_DIA','MAX_DIA_90','DROP_50','CP_DIA25','CP_DIA50','CP_DIA75','CP_DIA90','SKL_DEPTH','SKL_WIDTH']): currT=time.time() skel=Skeleton.Skeleton(imgL) testSkel,testDia=skel.skel(imgL) print 'Medial axis computed '+str(time.time()-currT)+'s' currT=time.time() path,skelGraph,crownT['DIA_STM'],skelSize=seg.findThickestPath(testSkel,testDia,xScale,yScale) allPara[counter][10]=skelSize print 'Central path computed '+str(time.time()-currT)+'s' if ifAnyKeyIsTrue(['TD_MED','TD_AVG','STA_RANGE','STA_DOM_I','STA_DOM_II','STA_25_I','STA_25_II','STA_50_I','STA_50_II','STA_75_I','STA_75_II','STA_90_I','STA_90_II','RTA_DOM_I','RTA_DOM_II','STA_MIN','STA_MAX','STA_MED','RTA_RANGE','RTA_MIN','RTA_MAX','RTA_MED','NR_RTP_SEG_I','NR_RTP_SEG_II','ADVT_COUNT','BASAL_COUNT','ADVT_ANG','BASAL_ANG','HYP_DIA','TAP_DIA','MAX_DIA_90','DROP_50','CP_DIA25','CP_DIA50','CP_DIA75','CP_DIA90','SKL_DEPTH','SKL_WIDTH','RTP_COUNT']): print 'Compute RTP skeleton' currT=time.time() rtpSkel,crownT['RTP_COUNT'], crownT['TD_MED'],crownT['TD_AVG'],crownT['MAX_DIA_90'], rtps, tips, crownT['SKL_WIDTH'], crownT['SKL_DEPTH'] =rtp.getRTPSkeleton(path,skelGraph,True) seg.setTips(tips) print 'RTP Skeleton computed '+str(time.time()-currT)+'s' allPara[len(allPara)-1][2]=seg.getFail() if ifAnyKeyIsTrue(['RDISTR_X','RDISTR_Y']): print 'Compute spatial root distribution' currT=time.time() crownT['RDISTR_X'],crownT['RDISTR_Y']=analysis.getSymmetry(rtps,rtpSkel) print 'Symmetry computed '+str(time.time()-currT)+'s' if rtpSkel!=-1: if ifAnyKeyIsTrue(['NR_RTP_SEG_I','NR_RTP_SEG_II','ADVT_COUNT','BASAL_COUNT','ADVT_ANG','BASAL_ANG','HYP_DIA','TAP_DIA']): print 'searching for hypocotyl' currT=time.time() branchRad,nrPaths=seg.findHypocotylCluster(path,rtpSkel) print 'hypocotyl computed '+str(time.time()-currT)+'s' print 'starting kmeans' try: currT=time.time() c1x,c1y,c2x,c2y = analysis.plotDiaRadius(nrPaths, branchRad,path,2) print '2 clusters computed in '+str(time.time()-currT)+'s' currT=time.time() segImg=seg.makeSegmentationPicture(path,rtpSkel,img,xScale,yScale,c1x,c1y,c2x,c2y) scipy.misc.imsave(io.getHomePath()+'/Result/' +io.getFileName()+ 'Seg2.png', segImg) crownT['ADVT_COUNT'],crownT['BASAL_COUNT'],crownT['NR_RTP_SEG_I'],crownT['NR_RTP_SEG_II'], crownT['HYP_DIA'], crownT['TAP_DIA'] =analysis.countRootsPerSegment(c1y,c2y,c1x,c2x) except: c1x=None c1y=None c2x=None c2y=None pass crownT['DROP_50']=analysis.RTPsOverDepth(path,rtpSkel) print 'count roots per segment' print 'Root classes computed in '+str(time.time()-currT)+'s' if ifAnyKeyIsTrue(['ADVT_ANG','BASAL_ANG','STA_RANGE','STA_DOM_I','STA_DOM_II','STA_25_I','STA_25_II','STA_50_I','STA_50_II','STA_75_I','STA_75_II','STA_90_I','STA_90_II','RTA_DOM_I','RTA_DOM_II','STA_MIN','STA_MAX','STA_MED','RTA_RANGE','RTA_MIN','RTA_MAX','RTA_MED']): currT=time.time() lat,corrBranchpts=seg.findLaterals(rtps, rtpSkel,(xScale+yScale)/2, None) print 'seg.findLaterals computed in '+str(time.time()-currT)+'s' print 'Compute angles at 2cm' currT=time.time() if c1x!=None and c1y!=None and c2x!=None and c2y!=None: crownT['ADVT_ANG'],crownT['BASAL_ANG']=analysis.anglesPerClusterAtDist(c1y, c2y, rtpSkel, path, lat, corrBranchpts, (xScale+yScale)/2, dist=20) else: crownT['ADVT_ANG']='nan' crownT['BASAL_NG']='nan' print 'angles at 2cm computed in '+str(time.time()-currT)+'s' if ifAnyKeyIsTrue(['STA_25_I','STA_25_II','STA_50_I','STA_50_II','STA_75_I','STA_75_II','STA_90_I','STA_90_II']): try: print 'compute quantile angles' currT=time.time() a25,a50,a75,a90=analysis.calculateAngleQuantiles(path,lat,corrBranchpts,rtpSkel) print 'angles computed in '+str(time.time()-currT)+'s' except: a25=['nan'] a50=['nan'] a75=['nan'] a90=['nan'] print 'ERROR: No quantile angles calculated' if ifAnyKeyIsTrue(['RTA_RANGE','RTA_MIN','RTA_MAX','RTA_MED']): try: print 'compute angles' currT=time.time() crownT['RTA_MED'],crownT['RTA_MIN'],crownT['RTA_MAX'],crownT['RTA_RANGE'],anglesN=analysis.calculateAngles(path,lat,corrBranchpts,rtpSkel) print 'RTA angle characteristics computed in '+str(time.time()-currT)+'s' except: print 'ERROR: No RTA angles calculated' if ifAnyKeyIsTrue(['STA_RANGE','STA_MIN','STA_MAX','STA_MED']): try: print 'compute STA angles' currT=time.time() crownT['STA_RANGE'],crownT['STA_MED'],crownT['STA_MIN'],crownT['STA_MAX'],angles=analysis.getLateralAngles(path,lat,corrBranchpts,rtpSkel) print 'STA angles characteristics computed in '+str(time.time()-currT)+'s' except: print 'ERROR: No STA angles calculated' if ifAnyKeyIsTrue(['CP_DIA25','CP_DIA50','CP_DIA75','CP_DIA90']): try: print 'compute diameter quantils' currT=time.time() crownT['CP_DIA25'],crownT['CP_DIA50'],crownT['CP_DIA75'],crownT['CP_DIA90']=analysis.getDiameterQuantilesAlongSinglePath(path,rtpSkel) print 'Tap diameters computed in '+str(time.time()-currT)+'s' except: print 'ERROR: No quantile diameters calculated' if ifAnyKeyIsTrue(['STA_DOM_I','STA_DOM_II']): try: print 'compute STA dominant angles' currT=time.time() crownT['STA_DOM_I'],crownT['STA_DOM_II']=analysis.findHistoPeaks(angles) print 'STA dominant angles computed in '+str(time.time()-currT)+'s' except: print 'ERROR: No dominant angles calculated (STA)' if ifAnyKeyIsTrue(['STA_25_I','STA_25_II']): try: currT=time.time() crownT['STA_25_I'],crownT['STA_25_II']=analysis.findHistoPeaks(a25) print 'STA 25 angles computed in '+str(time.time()-currT)+'s' except: print 'ERROR: No dominant angles25 calculated' if ifAnyKeyIsTrue(['STA_50_I','STA_50_II']): try: currT=time.time() crownT['STA_50_I'],crownT['STA_50_II']=analysis.findHistoPeaks(a50) print 'STA 50 angles computed in '+str(time.time()-currT)+'s' except: print 'ERROR: No dominant angles50 calculated' if ifAnyKeyIsTrue(['STA_75_I','STA_75_II']): try: currT=time.time() crownT['STA_75_I'],crownT['STA_75_II']=analysis.findHistoPeaks(a75) print 'STA 75 angles computed in '+str(time.time()-currT)+'s' except: print 'ERROR: No dominant angles75 calculated' if ifAnyKeyIsTrue(['STA_90_I','STA_90_II']): try: currT=time.time() crownT['STA_90_I'],crownT['STA_90_II']=analysis.findHistoPeaks(a90) print 'STA 90 angles computed in '+str(time.time()-currT)+'s' except: print 'ERROR: No dominant angles90 calculated' if ifAnyKeyIsTrue(['RTA_DOM_I','RTA_DOM_II']): try: currT=time.time() crownT['RTA_DOM_I'],crownT['RTA_DOM_II']=analysis.findHistoPeaks(anglesN) print 'angles computed in '+str(time.time()-currT)+'s' except: print 'ERROR: No dominant RTA angles calculated' io.setHomePath(oldHome) if maxExRoot >= 1: rtpSkel=-1 os.chdir(io.getHomePath()) io.setHomePath('./Lateral/') f=io.scanDir() for (counter,i) in enumerate(f): print 'processing lateral file: '+i if maxExRoot>0: xScale=allPara[counter/maxExRoot][7] yScale=allPara[counter/maxExRoot][8] io.setFileName(os.path.basename(i)) else: xScale=allPara[counter][7] yScale=allPara[counter][8] io.setFileName(os.path.basename(i)) io.setidIdx(counter) rtp=RootTipPaths.RootTipPaths(io) analysis=Analysis.Analysis(io,(xScale+yScale)/2) try: img=scipy.misc.imread(i,flatten=True) except: print 'Image not readable' img=[] pass if len(img)>0: seg=Segmentation.Segmentation(img,io=io) imgL=seg.label() if imgL!=None: skel=Skeleton.Skeleton(imgL) testSkel,testDia=skel.skel(imgL) path,skelGraph=seg.findThickestPathLateral(testSkel,testDia,xScale,yScale) if ifAnyKeyIsTrue(['LT_AVG_LEN','NODAL_LEN','LT_BRA_FRQ','NODAL_AVG_DIA','LT_AVG_ANG','LT_ANG_RANGE','LT_MIN_ANG','LT_MAX_ANG','LT_DIST_FIRST','LT_MED_DIA','LT_AVG_DIA']): rtpSkel,_,crownT['LT_MED_DIA'],crownT['LT_AVG_DIA'],_,rtps,_,_,_=rtp.getRTPSkeleton(path,skelGraph,True) if rtpSkel!=-1: if ifAnyKeyIsTrue(['LT_BRA_FRQ']): crownT['LT_BRA_FRQ']=analysis.getBranchingfrequencyAlongSinglePath(rtps,path) crownT['NODAL_AVG_DIA'],_=analysis.getDiametersAlongSinglePath(path,rtpSkel,(xScale+yScale)/2) crownT['NODAL_LEN']=analysis.getLengthOfPath(path) if ifAnyKeyIsTrue(['LT_DIST_FIRST','LT_AVG_LEN','LT_BRA_FRQ','LT_ANG_RANGE','LT_AVG_ANG','LT_MIN_ANG','LT_MAX_ANG']): lat,corrBranchpts,crownT['LT_DIST_FIRST']=seg.findLaterals(rtps, rtpSkel,(xScale+yScale)/2,path) if ifAnyKeyIsTrue(['LT_AVG_LEN']): crownT['LT_AVG_LEN']=analysis.getLateralLength(lat,path,rtpSkel) if ifAnyKeyIsTrue(['LT_ANG_RANGE','LT_AVG_ANG','LT_MIN_ANG','LT_MAX_ANG']): crownT['LT_ANG_RANGE'],crownT['LT_AVG_ANG'],crownT['LT_MIN_ANG'],crownT['LT_MAX_ANG'],_=analysis.getLateralAngles(path,lat,corrBranchpts,rtpSkel) allCrown.append(crownT.copy()) else: allCrown.append(crownT.copy()) io.setHomePath(oldHome)
def DiffusionConstants(self): with Parallel(n_jobs=self.threads) as parallel: self.LDValues = {} for key in self.bacteria.bacterium.keys(): self.LDValues[key] = Analysis.LDValues( self.bacteria.config[key]) # %% Linear - LogLog fullscale gp.c('reset') gp.c('set logscale xy 10') gp.c('set xlabel "{/Symbol t} (s)"') gp.c('set ylabel "MSD (m^2)"') gp.c('set key top left') gp.c("set terminal pngcairo enhanced" + " size 1600,1200 font 'ariel, 14'") amalg_dat_name = [] amalg_titles = [] for key in self.bacteria.bacterium.keys(): key_title = self.bacteria.config[key].name print('Started: %s \t Linear Analysis' % (key_title)) output = os.path.join(self.graph_dir, '%s_linear.png' % (key)) gp.c('set output "%s"' % (output)) g_title = 'Analysis of Linear Mean Squared Displacement - %s'\ % (key_title) gp.c('set title "%s"' % (g_title)) tau = Analysis.TauCalc(self.bacteria.config[key]) gp.c('set xrange [%f:%f]' % (tau.min() * 0.75, tau.max() * 1.25)) results_array = parallel( delayed(Analysis.Linear)(self.bacteria.bacterium[key] [bact], self.bacteria.config[key]) for bact in self.bacteria.bacterium[key].keys()) size = len(results_array) dat_name = os.path.join(self.plot_dir, '%s_msd_lin.dat' % (key)) graph_out = np.vstack((tau, results_array)) gp.s(graph_out, dat_name) title = ['notitle' for i in range(size)] plot_string = plotStringSingleFile(size, dat_name, 'with points', title) gp.c(plot_string) output = os.path.join(self.graph_dir, '%s_linear_mean.png' % (key)) gp.c('set output "%s"' % (output)) mean_results = np.mean(results_array, axis=0) std_dev = np.std(results_array, axis=0) std_error = std_dev / np.sqrt(size) current_results = np.vstack((tau, mean_results, std_error)) dat_name = os.path.join(self.plot_dir, '%s_msd_lin_mean.dat' % (key)) gp.s(current_results, dat_name) plot_string = 'plot "%s" u 1:2:3 with yerrorbars' % (dat_name) plot_string = plot_string + ' title "Mean Linear MSD"' gp.c(plot_string) amalg_dat_name.append(dat_name) amalg_titles.append('title "%s"' % (key_title)) print('Completed %s \t Linear Analysis' % (key)) amalg_formatting = 'u 1:2:3 with yerrorlines' amalg_plot_string = plotStringMultiFile(len(amalg_dat_name), amalg_dat_name, amalg_formatting, amalg_titles) output = os.path.join(self.graph_dir, 'linear_mean_amalg.png') gp.c('set output "%s"' % (output)) g_title = 'Analysis of Linear Mean Squared Displacement' gp.c('set title "%s"' % (g_title)) gp.c('set xrange [*:*]') gp.c(amalg_plot_string) # %% Linear - High Range (>1 sec) gp.c('reset') gp.c('set xlabel "{/Symbol t} (s)"') gp.c('set ylabel "MSD (m^2)"') gp.c('set key top left') gp.c("set terminal pngcairo enhanced" + " size 1600,1200 font 'ariel, 14'") amalg_dat_name = [] amalg_titles = [] amalg_grad = [] amalg_grad_titles = [] for key in self.bacteria.bacterium.keys(): line_colour = 1 key_title = self.bacteria.config[key].name print('Started: %s \t Linear Analysis High Range' % (key_title)) output = os.path.join(self.graph_dir, '%s_linear_hr.png' % (key)) gp.c('set output "%s"' % (output)) g_title = 'Analysis of Linear Mean Squared Displacement %s'\ % (key_title) gp.c('set title "%s"' % (g_title)) tau = Analysis.TauCalcHR(self.bacteria.config[key]) gp.c('set xrange [%f:%f]' % (tau.min() - 5, tau.max() + 5)) results_array = parallel( delayed(Analysis.LinearHighRange) (self.bacteria.bacterium[key][bact], self.bacteria.config[key]) for bact in self.bacteria.bacterium[key].keys()) size = len(results_array) dat_name = os.path.join(self.plot_dir, '%s_msd_lin_hr.dat' % (key)) graph_out = np.vstack((tau, results_array)) gp.s(graph_out, dat_name) title = ['notitle' for i in range(size)] plot_string = plotStringSingleFile(size, dat_name, 'with points', title) gp.c(plot_string) output = os.path.join(self.graph_dir, '%s_linear_mean_hr.png' % (key)) gp.c('set output "%s"' % (output)) mean_results = np.mean(results_array, axis=0) std_dev = np.std(results_array, axis=0) std_error = std_dev / np.sqrt(size) current_results = np.vstack((tau, mean_results, std_error)) gradient, y_intercept, r_value, p_value, grad_err\ = linregress(tau, mean_results) exp_diff = "Gradient = %.5e {/Symbol \261} %.5e" % (gradient, grad_err) r_2 = "R^2 = %f" % (r_value**2) fit_title = ("%s, %s, %s") % (key_title, exp_diff, r_2) dat_name = os.path.join(self.plot_dir, '%s_msd_lin_mean_hr.dat' % (key)) gp.s(current_results, dat_name) plot_string = 'plot "%s" u 1:2:3 with yerrorbars lc %d' %\ (dat_name, line_colour) plot_string = plot_string + ' title "%s - Mean MSD"'\ % (key_title) plot_string = plot_string + ', %e*x lc %d' %\ (gradient, line_colour) plot_string = plot_string + 'title "%s"' % (fit_title) gp.c(plot_string) amalg_dat_name.append(dat_name) amalg_titles.append('title "%s - Mean MSD" ' % (key_title)) amalg_grad.append(gradient) amalg_grad_titles.append(fit_title) print('Completed %s \t Linear Analysis High Range' % (key_title)) amalg_formatting = 'u 1:2:3 with yerrorbars' amalg_plot_string = plotStringMultiFileWithFit( len(amalg_dat_name), amalg_dat_name, amalg_formatting, amalg_titles, amalg_grad, amalg_grad_titles) output = os.path.join(self.graph_dir, 'linear_mean_amalg_hr.png') gp.c('set output "%s"' % (output)) g_title = 'Analysis of Linear Mean Squared Displacement' gp.c('set title "%s"' % (g_title)) gp.c('set xrange [*:*]') gp.c(amalg_plot_string) # %% Rotational Analysis amalg_dat_name = [] amalg_titles = [] gp.c('reset') gp.c('set logscale xy 10') gp.c('set xlabel "{/Symbol t} (s)"') gp.c('set ylabel "MSD ({/Symbol q}^2)"') gp.c('set key top left') gp.c("set terminal pngcairo enhanced" + " size 1600,1200 font 'ariel, 14'") for key in self.bacteria.bacterium.keys(): key_title = self.bacteria.config[key].name print("Started %s \t Rotational Analysis" % (key)) output = os.path.join(self.graph_dir, '%s_rotational.png' % (key)) gp.c('set output "%s"' % (output)) g_title = 'Analysis of Rotational Mean Squared '\ + 'Displacement - %s' % (key) gp.c('set title "%s" noenhanced' % (g_title)) tau = Analysis.TauCalc(self.bacteria.config[key]) gp.c('set xrange [%f:%f]' % (tau.min() * 0.75, tau.max() * 1.25)) results_array = parallel( delayed(Analysis.Rotational) (self.bacteria.bacterium[key][bact], self.bacteria.config[key]) for bact in self.bacteria.bacterium[key].keys()) size = len(results_array) dat_name = os.path.join(self.plot_dir, '%s_msd_rot.dat' % (key)) graph_out = np.vstack((tau, results_array)) gp.s(graph_out, dat_name) title = ['notitle' for i in range(size)] plot_string = plotStringSingleFile(size, dat_name, 'with points', title) gp.c(plot_string) output = os.path.join(self.graph_dir, '%s_rotational_mean.png' % (key)) gp.c('set output "%s"' % (output)) mean_results = np.mean(results_array, axis=0) std_dev = np.std(results_array, axis=0) std_error = std_dev / np.sqrt(size) current_results = np.vstack((tau, mean_results, std_error)) dat_name = os.path.join(self.plot_dir, '%s_msd_rot_mean.dat' % (key)) gp.s(current_results, dat_name) plot_string = 'plot "%s" u 1:2:3 with yerrorbars' % (dat_name) plot_string = plot_string + ' title "Mean Rotational MSD"' gp.c(plot_string) amalg_dat_name.append(dat_name) amalg_titles.append('title "%s"' % (key_title)) print("Completed %s \t Rotational Analysis" % (key)) amalg_formatting = 'u 1:2:3 with yerrorlines' amalg_plot_string = plotStringMultiFile(len(amalg_dat_name), amalg_dat_name, amalg_formatting, amalg_titles) output = os.path.join(self.graph_dir, 'rotational_mean_amalg.png') gp.c('set output "%s"' % (output)) g_title = 'Analysis of Rotational Mean Squared Displacement' gp.c('set title "%s"' % (g_title)) gp.c('set xrange [*:*]') gp.c(amalg_plot_string) # %% gp.c('reset') gp.c('set key top left') gp.c("set terminal pngcairo enhanced" + " size 1600,1200 font 'ariel, 14'") gp.c('unset logscale') gp.c('set ylabel "Probability Density"') gp.c('set xlabel "Run to Run Angle (degrees)"') amalg_dat_name = [] amalg_titles = [] for key in self.bacteria.bacterium.keys(): key_title = self.bacteria.config[key].name print("Started %s \t Run to Run angles" % (key)) if not self.bacteria.config[key].run_behaviour: print("%s is non-motile, ignoring" % (key)) continue if self.bacteria.config[key].archaea_mode: print("%s is an archaea, ignoring" % (key)) continue output = os.path.join(self.graph_dir, '%s_run_run_angle.png' % (key)) gp.c('set output "%s"' % (output)) title = 'Analysis of Run to Run Angle - %s'\ % (key) gp.c('set title "%s"' % (title)) angle_list = parallel( delayed(Analysis.RunRunAngles)(self.bacteria.cosines[key] [bact]) for bact in self.bacteria.cosines[key].keys()) angle_array = [] for i in range(len(angle_list)): angle_array = np.append(angle_array, angle_list[i]) angle_bins = np.linspace(0, np.pi, 40) angle_mean = np.mean(angle_array) angle_std = np.std(angle_array) angle_std_err = angle_std / np.sqrt(len(angle_array)) self.LDValues[key].avg_tumble = angle_mean self.LDValues[key].tumble_err = angle_std_err angle_med = np.median(angle_array) results, bin_edges = np.histogram(angle_array, bins=angle_bins, density=True) angle_points = np.delete(angle_bins, -1) + np.diff(angle_bins) / 2 graph_out = np.vstack((angle_points, results)) title = "%s, Mean = %6.2f, std. dev. = %6.2f, Median = %6.2f"\ % (key_title, angle_mean, angle_std, angle_med) dat_name = os.path.join(self.plot_dir, '%s_run_hist.dat' % (key)) gp.s(graph_out, dat_name) plot_string = 'plot "%s" u 1:2 with points title "%s"'\ % (dat_name, title) gp.c(plot_string) amalg_dat_name.append(dat_name) amalg_titles.append('title "%s"' % (title)) print("Completed %s \t Run to Run angles" % (key)) amalg_formatting = 'u 1:2 with points' if amalg_dat_name: amalg_plot_string = plotStringMultiFile( len(amalg_dat_name), amalg_dat_name, amalg_formatting, amalg_titles) output = os.path.join(self.graph_dir, 'run_run_angle_amalg.png') gp.c('set output "%s"' % (output)) g_title = 'Analysis of Run to Run Angles' gp.c('set title "%s"' % (g_title)) gp.c('set xrange [*:*]') gp.c(amalg_plot_string) # %% gp.c('reset') gp.c('set key top left') gp.c("set terminal pngcairo enhanced" + " size 1600,1200 font 'ariel, 14'") gp.c('set logscale y') gp.c('set ylabel "Probability Density"') gp.c('set xlabel "Run Duration (s)"') gp.c('set xrange [*:*]') amalg_dat_name = [] amalg_titles = [] for key in self.bacteria.bacterium.keys(): key_title = self.bacteria.config[key].name print("Started %s \t Run durations" % (key)) if not self.bacteria.config[key].run_behaviour: print("%s is non-motile, ignoring" % (key)) continue output = os.path.join(self.graph_dir, '%s_run_duration.png' % (key)) gp.c('set output "%s"' % (output)) title = 'Analysis of Run duration - %s'\ % (key) gp.c('set title "%s"' % (title)) time_list = parallel( delayed(Analysis.GetTimes) (self.bacteria.run_log[key][bact], self.bacteria.config[key]) for bact in self.bacteria.run_log[key].keys()) time_array = [] for i in range(len(time_list)): time_array = np.append(time_array, time_list[i]) time_mean = np.mean(time_array) time_std = np.std(time_array) time_std_err = time_std / (np.sqrt(len(time_array))) self.LDValues[key].avg_run_duration = time_mean self.LDValues[key].run_dur_err = time_std_err time_med = np.median(time_array) results, bin_edges = np.histogram(time_array, bins='auto', density=True) time_points = np.delete( bin_edges + np.mean(np.diff(bin_edges)) / 2, -1) graph_out = np.vstack((time_points, results)) title = "%s, Mean = %6.2f, std. dev. = %6.2f, Median = %6.2f"\ % (key_title, time_mean, time_std, time_med) dat_name = os.path.join(self.plot_dir, '%s_runduration_hist.dat' % (key)) gp.s(graph_out, dat_name) plot_string = 'plot "%s" u 1:2 with points title "%s"'\ % (dat_name, title) gp.c(plot_string) amalg_dat_name.append(dat_name) amalg_titles.append('title "%s"' % (title)) print("Completed %s \t Run duration" % (key)) if amalg_dat_name: amalg_formatting = 'u 1:2 with points' amalg_plot_string = plotStringMultiFile( len(amalg_dat_name), amalg_dat_name, amalg_formatting, amalg_titles) output = os.path.join(self.graph_dir, 'run_duration_amalg.png') gp.c('set output "%s"' % (output)) g_title = 'Analysis of Run duration' gp.c('set title "%s"' % (g_title)) gp.c('set xrange [*:*]') gp.c(amalg_plot_string) # %% gp.c('reset') gp.c('set key top left') gp.c("set terminal pngcairo enhanced" + " size 1600,1200 font 'ariel, 14'") gp.c('set logscale y') gp.c('set ylabel "Probability Density"') gp.c('set xlabel "Tumble Duration (s)"') gp.c('set xrange [*:*]') amalg_dat_name = [] amalg_titles = [] for key in self.bacteria.bacterium.keys(): key_title = self.bacteria.config[key].name print("Started %s \t Tumble durations" % (key)) if not self.bacteria.config[key].run_behaviour: print("%s is non-motile, ignoring" % (key)) continue if self.bacteria.config[key].archaea_mode: print("%s is an archaea, ignoring" % (key)) continue output = os.path.join(self.graph_dir, '%s_tumble_duration.png' % (key)) gp.c('set output "%s"' % (output)) title = 'Analysis of Tumble duration - %s'\ % (key_title) gp.c('set title "%s"' % (title)) time_list = parallel( delayed(Analysis.GetTimes) (self.bacteria.tumble_log[key][bact], self.bacteria.config[key]) for bact in self.bacteria.run_log[key].keys()) time_array = [] for i in range(len(time_list)): time_array = np.append(time_array, time_list[i]) time_mean = np.mean(time_array) self.LDValues[key].avg_tumble_duration = time_mean time_std = np.std(time_array) time_med = np.median(time_array) results, bin_edges = np.histogram(time_array, bins='auto', density=True) time_points = np.delete( bin_edges + np.mean(np.diff(bin_edges)) / 2, -1) graph_out = np.vstack((time_points, results)) title = "%s, Mean = %6.2f, std. dev. = %6.2f, Median = %6.2f"\ % (key_title, time_mean, time_std, time_med) dat_name = os.path.join(self.plot_dir, '%s_tumbleduration_hist.dat' % (key)) gp.s(graph_out, dat_name) plot_string = 'plot "%s" u 1:2 with points title "%s"'\ % (dat_name, title) gp.c(plot_string) amalg_dat_name.append(dat_name) amalg_titles.append('title "%s"' % (title)) print("Completed %s \t Tumble duration" % (key)) amalg_formatting = 'u 1:2 with points' if amalg_dat_name: amalg_plot_string = plotStringMultiFile( len(amalg_dat_name), amalg_dat_name, amalg_formatting, amalg_titles) output = os.path.join(self.graph_dir, 'tumble_duration_amalg.png') gp.c('set output "%s"' % (output)) g_title = 'Analysis of Tumble duration' gp.c('set title "%s"' % (g_title)) gp.c('set xrange [*:*]') gp.c(amalg_plot_string) LD_file = os.path.join(self.plot_dir, "LD_values.txt") with open(LD_file, "w") as LD_f: for key in self.bacteria.bacterium.keys(): if self.bacteria.config[key].run_behaviour: self.LDValues[key].LDCalc() write_string = "%s \t Name: %s \tLD Value: %e\n"\ % (key, self.bacteria.config[key].name, self.LDValues[key].LD_Diff) LD_f.write(write_string)
import numpy as np import Analysis import Import import BP print("正在训练神经网络") # 训练数据输入 TrainSet_x = Import.read_case('DataFile/StatusAndResult.xls', 'Sheet1_x') TrainSet_y = Import.read_case('DataFile/StatusAndResult.xls', 'Sheet2_y') # 训练神经网络 nt = BP.training(TrainSet_x, TrainSet_y) # 训练结果保存 Import.write_syn('DataFile/Syn.xls', nt.weights) # 测试数据输入 Test_input = np.array(Import.read_case('DataFile/Test.xls', 'Sheet1_x')) Test_output = Import.read_case('DataFile/Test.xls', 'Sheet2_y') # 测试数据输出 result = BP.forward(Test_input, nt)[nt.num_layers - 1] # 测试结果整理 result_sort = Analysis.sort(result) print("\n整理后的测试结果\n", result_sort) # 正确率计算 rate = Analysis.rate(result, Test_output) print("\n测试正确率为:\n", rate)
import Analysis import matplotlib as mpl from matplotlib import pyplot as plt import numpy as np import os run14 = Analysis.Run(np.load('2014_Raw.npy'), np.load('2014_Prune_strong.npy'), Analysis.bval2014, np.load('2014bf_new_prune_edit.npy')) run11 = Analysis.Run(np.load('2011_Raw.npy'), np.load('2011_Prune_strong_flat.npy'), Analysis.bval2011, np.load('2011bf_new_prune_flat_edit.npy')) i14 = run14.prune_data i11 = run11.prune_data print('Data loaded') os.chdir('fitcheck2019') print('Moved Dir') print('2014 Data') fig = plt.figure() n = 0 for i in i14: k = 0 b = Analysis.bval2014[n] for j in i.fits: i.plot() j.plot(False) plt.title('B:{},{}'.format(
def setup(): nexus = Nexus() problem = Data() nexus.optimization_problem = problem # ------------------------------------------------------------------- # Inputs # ------------------------------------------------------------------- # [ tag , initial, (lb,ub) , scaling , units ] problem.inputs = np.array([ [ 'wing_area' , 95 , ( 90. , 130. ) , 100. , Units.meter**2], [ 'cruise_altitude' , 11 , ( 9 , 14. ) , 10. , Units.km], ]) # ------------------------------------------------------------------- # Objective # ------------------------------------------------------------------- # throw an error if the user isn't specific about wildcards # [ tag, scaling, units ] problem.objective = np.array([ [ 'fuel_burn', 10000, Units.kg ] ]) # ------------------------------------------------------------------- # Constraints # ------------------------------------------------------------------- # [ tag, sense, edge, scaling, units ] problem.constraints = np.array([ [ 'design_range_fuel_margin' , '>', 0., 1E-1, Units.less], #fuel margin defined here as fuel ]) # ------------------------------------------------------------------- # Aliases # ------------------------------------------------------------------- # [ 'alias' , ['data.path1.name','data.path2.name'] ] problem.aliases = [ [ 'wing_area' , ['vehicle_configurations.*.wings.main_wing.areas.reference', 'vehicle_configurations.*.reference_area' ]], [ 'cruise_altitude' , 'missions.base.segments.climb_5.altitude_end' ], [ 'fuel_burn' , 'summary.base_mission_fuelburn' ], [ 'design_range_fuel_margin' , 'summary.max_zero_fuel_margin' ], ] # ------------------------------------------------------------------- # Vehicles # ------------------------------------------------------------------- # nexus.vehicle_configurations = Vehicles.setup() vehicle = plane.getPlane() nexus.vehicle_configurations = configurations.configSetup(vehicle) # ------------------------------------------------------------------- # Analyses # ------------------------------------------------------------------- nexus.analyses = Analysis.setup(nexus.vehicle_configurations) # ------------------------------------------------------------------- # Missions # ------------------------------------------------------------------- nexus.missions = Missions.setup(nexus.analyses) # ------------------------------------------------------------------- # Procedure # ------------------------------------------------------------------- nexus.procedure = Procedure.setup() # ------------------------------------------------------------------- # Summary # ------------------------------------------------------------------- nexus.summary = Data() nexus.total_number_of_iterations = 0 return nexus
contractJ1sql = "'Joe Biden'" contractJ2sql = "'Joe Biden'" marketJ2sql = "'Who will win the 2020 Democratic presidential nomination?'" marketJ1sql = "'Who will win the 2020 U.S. presidential election?'" contractJ1 = 'Joe Biden' contractJ2 = 'Joe Biden' marketJ2 = 'Who will win the 2020 Democratic presidential nomination?' marketJ1 = 'Who will win the 2020 U.S. presidential election?' csvfilenow = 'BidenAnalysis.csv' csvhist = 'biden.csv' buyaggros = 0.3 sellaggros = 0.3 zztop = 1 s1name = 'Joe Biden for President 2020' s2name = 'Joe Biden for Democratic Nomination 2020' # In[ ]: an.analysis_loop(csvhist, csvfilenow, marketJ1sql, marketJ2sql,contractJ1sql, contractJ2sql, marketJ1, marketJ2, contractJ1, contractJ2, buyaggros, sellaggros, zztop, s1name, s2name) # In[4]:
import os if os.environ.has_key('REPack'): sys.path.append(os.environ['REPack']) else: sys.path.append(r'..\\') import pprint import logging import json import Analysis logging.basicConfig(level=logging.DEBUG) logger = logging.getLogger(__name__) if __name__ == '__main__': disasm = Analysis.Disasm() save_filename = '' if len(disasm.Args) > 0: save_filename = disasm.Args[0] else: import UI global form title = 'Breakpoints-UI' try: form form.OnClose(form) form = UI.Form(title) except:
import Analysis, sys, math, numpy as np, pylab mapping = Analysis.loadmapping(sys.argv[1]) num_rows = 256 num_cols = 512 colors = np.zeros((256,512,3)) starty = np.random.rand(1,100)*150 + 50 startx = np.random.rand(1,100)*400 + 50 endx, endy = mapping(startx, starty) base_shift_x = 256 # (endx - startx).mean() base_shift_y = 512 # (endy - starty).mean() for x in range(num_cols): for y in range(num_rows): newx, newy = mapping(x,y) sat = 1 hue = (90 + 180 / math.pi * math.atan2(newy - base_shift_y, newx - base_shift_x)) % 360 val = min(.001 * math.sqrt((newy - base_shift_y)**2 + (newx - base_shift_x)**2), 1) hueprm = hue / 60 C = val * sat X = C * (1 - abs((hueprm %2) -1 )) if hueprm < 1: r1, g1, b1 = C, X, 0 elif hueprm < 2: r1, g1, b1 = X, C, 0 elif hueprm < 3: r1, g1, b1 = X, C, 0
# def __init__(self,conf, game10,game20,modifier='nothing',verbose0=False): ThisPebble = PB.Pebbles(ThisConf, 3, 3, 'nothing', False) # play game ThisPebble.play_game() # compute rigid clusters cidx, clusterall, clusterallBonds, clusteridx, BigCluster = ThisPebble.rigid_cluster( ) ########### Setting up the dynamical matrix and getting eigenmodes # This itself does very little, just creates an empty Hessian class # __init__(self,conf0): ThisHessian = HS.Hessian(ThisConf) ########## Have a look at some analysis functions of the rigid clusters #def __init__(self,conf0,pebbles0,hessian0,verbose=False): ThisAnalysis = AN.Analysis(ThisConf, ThisPebble, ThisHessian, 0.01, False) # stress statistics zav, nm, pres, fxbal, fybal, torbal, mobin, mohist, sxx, syy, sxy, syx = ThisAnalysis.getStressStat( ) # cluster statistics frac, fracmax, lenx, leny = ThisAnalysis.clusterStatistics() #def plotStresses(self,plotCir,plotVel,plotCon,plotF,plotStress,**kwargs): fig1 = ThisAnalysis.plotStresses(True, False, False, True, False) #def plotPebbles(self,plotCir,plotPeb,plotPebCon,plotClus,plotOver,**kwargs): #ThisAnalysis.plotPebbles(True,True,True,False,False) fig2 = ThisAnalysis.plotPebbles(True, True, False, True, False) ######### continuing with the Hessian now # constructing the matrix # makeHessian(self,frictional,recomputeFnor,stabilise,verbose=False): ThisHessian.makeHessian(True, False, 0, True)
import Analysis import math import json import heapq import numpy as np top_k = 1000 ############################# word_list = [] avg = sum(Analysis.vectorizer.idf_) / Analysis.vectorizer.idf_.__len__() scores = Analysis.get_scores() print("Analysis done....") writer1 = open('scores.txt', 'w', encoding="utf8") writer = open('words.txt', 'w', encoding="utf8") def cosine_similarity(v1, v2): "compute cosine similarity of v1 to v2: (v1 dot v2)/{||v1||*||v2||)" sumxx, sumxy, sumyy = 0, 0, 0 for i in range(len(v1)): x = v1[i] y = v2[i] sumxx += x * x sumyy += y * y sumxy += x * y
help = "Maximum allowed inter-color colocalization distance", default = 100, type = "float") parser.add_option('-q', '--quiet', dest = "quiet", action = "store_true", default = False) parser.add_option('-d', '--quiver-distance', dest="quiv_dist", help = "Minimum distance between points on the quiver map", default = 10) opts, args = parser.parse_args() if not opts.map: if opts.x1 and opts.x2 and opts.y1 and opts.y2: opts.xshift = opts.x2 - opts.x1 opts.yshift = opts.y2 - opts.y1 else: mapping = Analysis.loadmapping(opts.map) if opts.map2: mapping2 = Analysis.loadmapping(opts.map2) if len(args) == 1 and '*' in args[0]: args = glob(args[0]) print "Found this many files", len(args) xrs = [] xls = [] yrs = [] yls = [] varxrs = [] varxls = [] for fname in args:
oscars_winloss[y] = Player3.get_wins() - Player3.get_losses() safe_winloss[y] = Player4.get_wins() - Player4.get_losses() martingale_streak[y] = Player1.get_win_streak() manhattan_streak[y] = Player2.get_win_streak() oscars_streak[y] = Player3.get_win_streak() safe_streak[y] = Player4.get_win_streak() # reset all players to defaults Player1.__init__(p1_pot, min_bet, p1_wins, p1_losses, win_streak, profit) Player2.__init__(p2_pot, min_bet, p2_wins, p2_losses, win_streak, profit) Player3.__init__(p3_pot, min_bet, p3_wins, p3_losses, win_streak, profit) Player4.__init__(p4_pot, min_bet, p4_wins, p4_losses, win_streak, profit) Dealer.__init__(d_pot, d_wins, d_losses) # conduct analysis # generate analysis objects Martingale_Stats = Analysis.AnalyticRecords(martingale_pot, martingale_winloss, martingale_streak) Manhattan_Stats = Analysis.AnalyticRecords(manhattan_pot, manhattan_winloss, manhattan_streak) Oscar_Stats = Analysis.AnalyticRecords(oscars_pot, oscars_winloss, oscars_streak) Safe_Stats = Analysis.AnalyticRecords(safe_pot, safe_winloss, safe_streak) # part A - generate descriptive statistics and build confidence intervals z_val = calc_z_val(alpha) # mean, variance, and 95% confidence intervals for each strategy # martingale strategy print("Martingale strategy pot: mean " + str(Martingale_Stats.get_pot_mean()) + ", var " + str(Martingale_Stats.get_pot_var())) print("Martingale strategy win/loss: mean " + str(Martingale_Stats.get_winloss_mean()) + ", var " + str(Martingale_Stats.get_winloss_var())) print("Martingale strategy streak: mean " +
print 'Processing trajectory...', sampler.traj.process() # compute averages, etc. print '...Done.' print 'Writing results...', sampler.traj.write_results(os.path.join(outdir,'traj_lambda%2.2f.npz'%lam)) print '...Done.' sampler.traj.read_results(os.path.join(outdir,'traj_lambda%2.2f.npz'%lam)) print 'Pickling the sampler object ...', outfilename = 'sampler_lambda%2.2f.pkl'%lam print outfilename, fout = open(os.path.join(outdir, outfilename), 'wb') # Pickle dictionary using protocol 0. cPickle.dump(sampler, fout) fout.close() print '...Done.' ######################################### # Let's do analysis using MBAR and plot figures ############ MBAR and Figures ########### # Specify necessary argument values A = Analysis(100,dataFiles,outdir) A.plot()
def setUp(self): self.analysis = Analysis.Analysis()
class DisassemblerView(QAbstractScrollArea): statusUpdated = Signal(QWidget, name="statusUpdated") def __init__(self, data, filename, view, parent): super(DisassemblerView, self).__init__(parent) self.status = "" self.view = view self.data = data for type in ExeFormats: exe = type(data) if exe.valid: self.data = exe self.view.exe = exe break # Create analysis and start it in another thread self.analysis = Analysis(self.data) self.analysis_thread = threading.Thread(None, self.analysis_thread_proc) self.analysis_thread.daemon = True self.analysis_thread.start() # Start disassembly view at the entry point of the binary if hasattr(self.data, "entry"): self.function = self.data.entry() else: self.function = None self.update_id = None self.ready = False self.desired_pos = None self.highlight_token = None self.cur_instr = None self.scroll_mode = False self.blocks = {} self.show_il = False self.simulation = None # Create timer to automatically refresh view when it needs to be updated self.updateTimer = QTimer() self.updateTimer.setInterval(100) self.updateTimer.setSingleShot(False) self.updateTimer.timeout.connect(self.updateTimerEvent) self.updateTimer.start() self.initFont() # Initialize scroll bars self.width = 0 self.height = 0 self.setHorizontalScrollBarPolicy(Qt.ScrollBarAsNeeded) self.setVerticalScrollBarPolicy(Qt.ScrollBarAsNeeded) self.horizontalScrollBar().setSingleStep(self.charWidth) self.verticalScrollBar().setSingleStep(self.charHeight) areaSize = self.viewport().size() self.adjustSize(areaSize.width(), areaSize.height()) # Setup navigation self.view.register_navigate("disassembler", self, self.navigate) self.view.register_navigate("make_proc", self, self.make_proc) self.search_regex = None self.last_search_type = FindDialog.SEARCH_HEX def initFont(self): # Get font and compute character sizes self.font = getMonospaceFont() self.baseline = int(QFontMetricsF(self.font).ascent()) self.charWidth = QFontMetricsF(self.font).width('X') self.charHeight = int(QFontMetricsF(self.font).height()) + getExtraFontSpacing() self.charOffset = getFontVerticalOffset() def adjustSize(self, width, height): # Recompute size information self.renderWidth = self.width self.renderHeight = self.height self.renderXOfs = 0 self.renderYOfs = 0 if self.renderWidth < width: self.renderXOfs = int((width - self.renderWidth) / 2) self.renderWidth = width if self.renderHeight < height: self.renderYOfs = int((height - self.renderHeight) / 2) self.renderHeight = height # Update scroll bar information self.horizontalScrollBar().setPageStep(width) self.horizontalScrollBar().setRange(0, self.renderWidth - width) self.verticalScrollBar().setPageStep(height) self.verticalScrollBar().setRange(0, self.renderHeight - height) def resizeEvent(self, event): # Window was resized, adjust scroll bar self.adjustSize(event.size().width(), event.size().height()) def get_cursor_pos(self): if self.cur_instr is None: return self.function return self.cur_instr def set_cursor_pos(self, addr): if not self.view.navigate("disassembler", addr): self.view_in_hex_editor(addr) def get_selection_range(self): return (self.get_cursor_pos(), self.get_cursor_pos()) def set_selection_range(self, begin, end): self.set_cursor_pos(begin) def write(self, data): pos = self.get_cursor_pos() if pos is None: return False return self.data.write(pos, data) == len(data) def copy_address(self): clipboard = QApplication.clipboard() clipboard.clear() mime = QMimeData() mime.setText("0x%x" % self.get_cursor_pos()) clipboard.setMimeData(mime) def analysis_thread_proc(self): self.analysis.analyze() def closeRequest(self): # Stop analysis when closing tab self.analysis.stop() return True def paintEvent(self, event): # Initialize painter p = QPainter(self.viewport()) p.setFont(self.font) xofs = self.horizontalScrollBar().value() yofs = self.verticalScrollBar().value() if not self.ready: # Analysis for the current function is not yet complete, paint loading screen gradient = QLinearGradient(QPointF(0, 0), QPointF(self.viewport().size().width(), self.viewport().size().height())) gradient.setColorAt(0, QColor(232, 232, 232)) gradient.setColorAt(1, QColor(192, 192, 192)) p.setPen(QColor(0, 0, 0, 0)) p.setBrush(QBrush(gradient)) p.drawRect(0, 0, self.viewport().size().width(), self.viewport().size().height()) if self.function is None: text = "No function selected" else: text = "Loading..." p.setPen(Qt.black) p.drawText((self.viewport().size().width() / 2) - ((len(text) * self.charWidth) / 2), (self.viewport().size().height() / 2) + self.charOffset + self.baseline - (self.charHeight / 2), text) return # Render background gradient = QLinearGradient(QPointF(-xofs, -yofs), QPointF(self.renderWidth - xofs, self.renderHeight - yofs)) gradient.setColorAt(0, QColor(232, 232, 232)) gradient.setColorAt(1, QColor(192, 192, 192)) p.setPen(QColor(0, 0, 0, 0)) p.setBrush(QBrush(gradient)) p.drawRect(0, 0, self.viewport().size().width(), self.viewport().size().height()) p.translate(self.renderXOfs - xofs, self.renderYOfs - yofs) # Render each node for block in self.blocks.values(): # Render shadow p.setPen(QColor(0, 0, 0, 0)) p.setBrush(QColor(0, 0, 0, 128)) p.drawRect(block.x + self.charWidth + 4, block.y + self.charWidth + 4, block.width - (4 + 2 * self.charWidth), block.height - (4 + 2 * self.charWidth)) # Render node background gradient = QLinearGradient(QPointF(0, block.y + self.charWidth), QPointF(0, block.y + block.height - self.charWidth)) gradient.setColorAt(0, QColor(255, 255, 252)) gradient.setColorAt(1, QColor(255, 255, 232)) p.setPen(Qt.black) p.setBrush(QBrush(gradient)) p.drawRect(block.x + self.charWidth, block.y + self.charWidth, block.width - (4 + 2 * self.charWidth), block.height - (4 + 2 * self.charWidth)) if self.cur_instr != None: y = block.y + (2 * self.charWidth) + (len(block.block.header_text.lines) * self.charHeight) for instr in block.block.instrs: if instr.addr == self.cur_instr: p.setPen(QColor(0, 0, 0, 0)) p.setBrush(QColor(255, 255, 128, 128)) p.drawRect(block.x + self.charWidth + 3, y, block.width - (10 + 2 * self.charWidth), len(instr.text.lines) * self.charHeight) y += len(instr.text.lines) * self.charHeight if self.highlight_token: # Render highlighted tokens x = block.x + (2 * self.charWidth) y = block.y + (2 * self.charWidth) for line in block.block.header_text.tokens: for token in line: if token[2:] == self.highlight_token: p.setPen(QColor(0, 0, 0, 0)) p.setBrush(QColor(192, 0, 0, 64)) p.drawRect(x + token[0] * self.charWidth, y, token[1] * self.charWidth, self.charHeight) y += self.charHeight for instr in block.block.instrs: for line in instr.text.tokens: for token in line: if token[2:] == self.highlight_token: p.setPen(QColor(0, 0, 0, 0)) p.setBrush(QColor(192, 0, 0, 64)) p.drawRect(x + token[0] * self.charWidth, y, token[1] * self.charWidth, self.charHeight) y += self.charHeight # Render node text x = block.x + (2 * self.charWidth) y = block.y + (2 * self.charWidth) for line in block.block.header_text.lines: partx = x for part in line: p.setPen(part[1]) p.drawText(partx, y + self.charOffset + self.baseline, part[0]) partx += len(part[0]) * self.charWidth y += self.charHeight for instr in block.block.instrs: for line in instr.text.lines: partx = x for part in line: p.setPen(part[1]) p.drawText(partx, y + self.charOffset + self.baseline, part[0]) partx += len(part[0]) * self.charWidth y += self.charHeight # Render edges for edge in block.edges: p.setPen(edge.color) p.setBrush(edge.color) p.drawPolyline(edge.polyline) p.drawConvexPolygon(edge.arrow) def isMouseEventInBlock(self, event): # Convert coordinates to system used in blocks xofs = self.horizontalScrollBar().value() yofs = self.verticalScrollBar().value() x = event.x() + xofs - self.renderXOfs y = event.y() + yofs - self.renderYOfs # Check each block for hits for block in self.blocks.values(): # Compute coordinate relative to text area in block blockx = x - (block.x + (2 * self.charWidth)) blocky = y - (block.y + (2 * self.charWidth)) # Check to see if click is within bounds of block if (blockx < 0) or (blockx > (block.width - 4 * self.charWidth)): continue if (blocky < 0) or (blocky > (block.height - 4 * self.charWidth)): continue return True return False def getInstrForMouseEvent(self, event): # Convert coordinates to system used in blocks xofs = self.horizontalScrollBar().value() yofs = self.verticalScrollBar().value() x = event.x() + xofs - self.renderXOfs y = event.y() + yofs - self.renderYOfs # Check each block for hits for block in self.blocks.values(): # Compute coordinate relative to text area in block blockx = x - (block.x + (2 * self.charWidth)) blocky = y - (block.y + (2 * self.charWidth)) # Check to see if click is within bounds of block if (blockx < 0) or (blockx > (block.width - 4 * self.charWidth)): continue if (blocky < 0) or (blocky > (block.height - 4 * self.charWidth)): continue # Compute row within text row = int(blocky / self.charHeight) # Determine instruction for this row cur_row = len(block.block.header_text.lines) if row < cur_row: return block.block.entry for instr in block.block.instrs: if row < cur_row + len(instr.text.lines): return instr.addr cur_row += len(instr.text.lines) return None def getTokenForMouseEvent(self, event): # Convert coordinates to system used in blocks xofs = self.horizontalScrollBar().value() yofs = self.verticalScrollBar().value() x = event.x() + xofs - self.renderXOfs y = event.y() + yofs - self.renderYOfs # Check each block for hits for block in self.blocks.values(): # Compute coordinate relative to text area in block blockx = x - (block.x + (2 * self.charWidth)) blocky = y - (block.y + (2 * self.charWidth)) # Check to see if click is within bounds of block if (blockx < 0) or (blockx > (block.width - 4 * self.charWidth)): continue if (blocky < 0) or (blocky > (block.height - 4 * self.charWidth)): continue # Compute row and column within text col = int(blockx / self.charWidth) row = int(blocky / self.charHeight) # Check tokens to see if one was clicked cur_row = 0 for line in block.block.header_text.tokens: if cur_row == row: for token in line: if (col >= token[0]) and (col < (token[0] + token[1])): # Clicked on a token return token cur_row += 1 for instr in block.block.instrs: for line in instr.text.tokens: if cur_row == row: for token in line: if (col >= token[0]) and (col < (token[0] + token[1])): # Clicked on a token return token cur_row += 1 return None def find_instr(self, addr): for block in self.blocks.values(): for instr in block.block.instrs: if instr.addr == addr: return instr return None def nop_out(self, addr): instr = self.find_instr(addr) if instr != None: self.view.begin_undo() instr.patch_to_nop(self.data) self.view.commit_undo() def always_branch(self, addr): instr = self.find_instr(addr) if instr != None: self.view.begin_undo() instr.patch_to_always_branch(self.data) self.view.commit_undo() def invert_branch(self, addr): instr = self.find_instr(addr) if instr != None: self.view.begin_undo() instr.patch_to_invert_branch(self.data) self.view.commit_undo() def skip_and_return_zero(self, addr): instr = self.find_instr(addr) if instr != None: self.view.begin_undo() instr.patch_to_zero_return(self.data) self.view.commit_undo() def skip_and_return_value(self, addr): instr = self.find_instr(addr) if instr != None: value, ok = QInputDialog.getText(self, "Skip and Return Value", "Return value:", QLineEdit.Normal) if ok: try: value = int(value, 0) except: QMessageBox.critical(self, "Error", "Expected numerical address") return self.view.begin_undo() instr.patch_to_fixed_return_value(self.data, value) self.view.commit_undo() def view_in_hex_editor(self, addr): if not self.view.navigate("exe", addr): self.view.navigate("hex", addr) def show_address(self): if "address" in self.analysis.options: addr = False else: addr = True self.analysis.set_address_view(addr) def context_menu(self, addr): popup = QMenu() view_in_hex = popup.addAction("View in &hex editor") view_in_hex.triggered.connect(lambda : self.view_in_hex_editor(addr)) view_in_hex.setShortcut(QKeySequence(Qt.Key_H)) popup.addAction("Copy address", self.copy_address) enter_name_action = popup.addAction("Re&name symbol", self.enter_name) enter_name_action.setShortcut(QKeySequence(Qt.Key_N)) undefine_name_action = popup.addAction("&Undefine symbol", self.undefine_name) undefine_name_action.setShortcut(QKeySequence(Qt.Key_U)) show_address_action = popup.addAction("Show &address", self.show_address) show_address_action.setCheckable(True) show_address_action.setChecked("address" in self.analysis.options) popup.addSeparator() patch = popup.addMenu("&Patch") patch.addAction("Convert to NOP").triggered.connect(lambda : self.nop_out(addr)) instr = self.find_instr(addr) if instr: if instr.is_patch_branch_allowed(): patch.addAction("Never branch").triggered.connect(lambda : self.nop_out(addr)) patch.addAction("Always branch").triggered.connect(lambda : self.always_branch(addr)) patch.addAction("Invert branch").triggered.connect(lambda : self.invert_branch(addr)) if instr.is_patch_to_zero_return_allowed(): patch.addAction("Skip and return zero").triggered.connect(lambda : self.skip_and_return_zero(addr)) if instr.is_patch_to_fixed_return_value_allowed(): patch.addAction("Skip and return value...").triggered.connect(lambda : self.skip_and_return_value(addr)) popup.exec_(QCursor.pos()) def mousePressEvent(self, event): if (event.button() != Qt.LeftButton) and (event.button() != Qt.RightButton): return if not self.isMouseEventInBlock(event): # Click outside any block, enter scrolling mode self.scroll_base_x = event.x() self.scroll_base_y = event.y() self.scroll_mode = True self.viewport().grabMouse() return # Check for click on a token and highlight it token = self.getTokenForMouseEvent(event) if token: self.highlight_token = token[2:] else: self.highlight_token = None # Update current instruction instr = self.getInstrForMouseEvent(event) if instr != None: self.cur_instr = instr else: self.cur_instr = None self.viewport().update() if (instr != None) and (event.button() == Qt.RightButton): self.context_menu(instr) def mouseMoveEvent(self, event): if self.scroll_mode: x_delta = self.scroll_base_x - event.x() y_delta = self.scroll_base_y - event.y() self.scroll_base_x = event.x() self.scroll_base_y = event.y() self.horizontalScrollBar().setValue(self.horizontalScrollBar().value() + x_delta) self.verticalScrollBar().setValue(self.verticalScrollBar().value() + y_delta) def mouseReleaseEvent(self, event): if event.button() != Qt.LeftButton: return if self.scroll_mode: self.scroll_mode = False self.viewport().releaseMouse() def mouseDoubleClickEvent(self, event): token = self.getTokenForMouseEvent(event) if token and (token[2] == "ptr"): self.analysis.lock.acquire() if not self.analysis.functions.has_key(token[3]): # Not a function or not analyzed, go to address in hex editor addr = token[3] self.analysis.lock.release() self.view_in_hex_editor(addr) else: self.view.add_history_entry() self.function = token[3] self.ready = False self.desired_pos = None self.cur_instr = None self.highlight_token = None self.viewport().update() self.analysis.lock.release() def go_to_address(self): addr_str, ok = QInputDialog.getText(self, "Go To Address", "Address:", QLineEdit.Normal) if ok: try: addr = int(addr_str, 16) if (addr < self.data.start()) or (addr > self.data.end()): if hasattr(self.data, "symbols_by_name") and (addr_str in self.data.symbols_by_name): addr = self.data.symbols_by_name[addr_str] else: QMessageBox.critical(self, "Error", "Address out of range") return except: if hasattr(self.data, "symbols_by_name") and (addr_str in self.data.symbols_by_name): addr = self.data.symbols_by_name[addr_str] elif (addr_str[0] == '@') and hasattr(self.data, "symbols_by_name") and (addr_str[1:] in self.data.symbols_by_name): addr = self.data.symbols_by_name[addr_str[1:]] else: QMessageBox.critical(self, "Error", "Invalid address or symbol") return # Try navigating within disassembly, if it isn't within a function then # navigate to the hex editor if not self.view.navigate("disassembler", addr): self.view_in_hex_editor(addr) def enter_name(self): # A symbol must be selected if (self.highlight_token == None) or (self.highlight_token[0] != "ptr"): QMessageBox.critical(self, "Error", "No symbol selected.") return addr = self.highlight_token[1] name = self.highlight_token[2] # Ask for new name new_name, ok = QInputDialog.getText(self, "Rename Symbol", "Symbol name:", QLineEdit.Normal, name) if ok: self.analysis.create_symbol(addr, new_name) def undefine_name(self): # A symbol must be selected if (self.highlight_token == None) or (self.highlight_token[0] != "ptr"): QMessageBox.critical(self, "Error", "No symbol selected.") return addr = self.highlight_token[1] name = self.highlight_token[2] # Ask for new name self.analysis.undefine_symbol(addr, name) def navigate_for_find(self, addr): func, instr = self.analysis.find_instr(addr, True) if func != None: self.navigate(addr) else: self.make_proc(addr) self.cur_instr = addr self.desired_pos = None def perform_find(self, dlg): self.search_regex = dlg.search_regex() if self.cur_instr != None: self.search_start = self.cur_instr else: if self.function is None: return self.search_start = self.function found_loc = self.data.find(self.search_regex, self.search_start) if found_loc != -1: self.view.add_history_entry() self.navigate_for_find(found_loc) self.search_pos = found_loc + 1 return found_loc = self.data.find(self.search_regex, self.data.start()) if (found_loc != -1) and (found_loc < self.search_start): self.view.add_history_entry() self.navigate_for_find(found_loc) self.search_pos = found_loc + 1 return QMessageBox.information(self, "Not Found", "Search string not found.") def find(self): dlg = FindDialog(self.last_search_type, self) if dlg.exec_() == QDialog.Accepted: self.last_search_type = dlg.search_type() self.perform_find(dlg) def find_next(self): if self.search_regex == None: QMessageBox.critical(self, "Error", "No active search") return found_loc = self.data.find(self.search_regex, self.search_pos) if self.search_pos >= self.search_start: if found_loc != -1: self.view.add_history_entry() self.navigate_for_find(found_loc) self.search_pos = found_loc + 1 return self.search_pos = 0 else: if (found_loc != -1) and (found_loc < self.search_start): self.view.add_history_entry() self.navigate_for_find(found_loc) self.search_pos = found_loc + 1 return QMessageBox.information(self, "End of Search", "No additional matches found.") self.search_pos = self.search_start return found_loc = self.data.find(self.search_regex, self.search_pos) if found_loc < self.search_start: self.view.add_history_entry() self.navigate_for_find(found_loc) self.search_pos = found_loc + 1 return QMessageBox.information(self, "End of Search", "No additional matches found.") self.search_pos = self.search_start def keyPressEvent(self, event): if event.key() == Qt.Key_H: if self.cur_instr != None: self.view_in_hex_editor(self.cur_instr) else: if self.function is not None: self.view_in_hex_editor(self.function) elif event.key() == Qt.Key_G: self.go_to_address() elif event.key() == Qt.Key_N: self.enter_name() elif event.key() == Qt.Key_U: self.undefine_name() elif event.key() == Qt.Key_Slash: dlg = FindDialog(FindDialog.SEARCH_REGEX, self) if dlg.exec_() == QDialog.Accepted: self.perform_find(dlg) else: super(DisassemblerView, self).keyPressEvent(event) def prepareGraphNode(self, block): # Compute size of node in pixels width = 0 height = 0 for line in block.block.header_text.lines: chars = 0 for part in line: chars += len(part[0]) if chars > width: width = chars height += 1 for instr in block.block.instrs: for line in instr.text.lines: chars = 0 for part in line: chars += len(part[0]) if chars > width: width = chars height += 1 block.width = (width + 4) * self.charWidth + 4 block.height = (height * self.charHeight) + (4 * self.charWidth) + 4 def adjustGraphLayout(self, block, col, row): block.col += col block.row += row for edge in block.new_exits: self.adjustGraphLayout(self.blocks[edge], col, row) def computeGraphLayout(self, block): # Compute child node layouts and arrange them horizontally col = 0 row_count = 1 for edge in block.new_exits: self.computeGraphLayout(self.blocks[edge]) self.adjustGraphLayout(self.blocks[edge], col, 1) col += self.blocks[edge].col_count if (self.blocks[edge].row_count + 1) > row_count: row_count = self.blocks[edge].row_count + 1 block.row = 0 if col >= 2: # Place this node centered over the child nodes block.col = (col - 2) / 2 block.col_count = col else: # No child nodes, set single node's width (nodes are 2 columns wide to allow # centering over a branch) block.col = 0 block.col_count = 2 block.row_count = row_count def isEdgeMarked(self, edges, row, col, index): if index >= len(edges[row][col]): return False return edges[row][col][index] def markEdge(self, edges, row, col, index): while len(edges[row][col]) <= index: edges[row][col] += [False] edges[row][col][index] = True def findHorizEdgeIndex(self, edges, row, min_col, max_col): # Find a valid index i = 0 while True: valid = True for col in range(min_col, max_col + 1): if self.isEdgeMarked(edges, row, col, i): valid = False break if valid: break i += 1 # Mark chosen index as used for col in range(min_col, max_col + 1): self.markEdge(edges, row, col, i) return i def findVertEdgeIndex(self, edges, col, min_row, max_row): # Find a valid index i = 0 while True: valid = True for row in range(min_row, max_row + 1): if self.isEdgeMarked(edges, row, col, i): valid = False break if valid: break i += 1 # Mark chosen index as used for row in range(min_row, max_row + 1): self.markEdge(edges, row, col, i) return i def routeEdge(self, horiz_edges, vert_edges, edge_valid, start, end, color): edge = DisassemblerEdge(color, end) # Find edge index for initial outgoing line i = 0 while True: if not self.isEdgeMarked(vert_edges, start.row + 1, start.col + 1, i): break i += 1 self.markEdge(vert_edges, start.row + 1, start.col + 1, i) edge.addPoint(start.row + 1, start.col + 1) edge.start_index = i horiz = False # Find valid column for moving vertically to the target node if end.row < (start.row + 1): min_row = end.row max_row = start.row + 1 else: min_row = start.row + 1 max_row = end.row col = start.col + 1 if min_row != max_row: ofs = 0 while True: col = start.col + 1 - ofs if col >= 0: valid = True for row in range(min_row, max_row + 1): if not edge_valid[row][col]: valid = False break if valid: break col = start.col + 1 + ofs if col < len(edge_valid[min_row]): valid = True for row in range(min_row, max_row + 1): if not edge_valid[row][col]: valid = False break if valid: break ofs += 1 if col != (start.col + 1): # Not in same column, need to generate a line for moving to the correct column if col < (start.col + 1): min_col = col max_col = start.col + 1 else: min_col = start.col + 1 max_col = col index = self.findHorizEdgeIndex(horiz_edges, start.row + 1, min_col, max_col) edge.addPoint(start.row + 1, col, index) horiz = True if end.row != (start.row + 1): # Not in same row, need to generate a line for moving to the correct row index = self.findVertEdgeIndex(vert_edges, col, min_row, max_row) edge.addPoint(end.row, col, index) horiz = False if col != (end.col + 1): # Not in ending column, need to generate a line for moving to the correct column if col < (end.col + 1): min_col = col max_col = end.col + 1 else: min_col = end.col + 1 max_col = col index = self.findHorizEdgeIndex(horiz_edges, end.row, min_col, max_col) edge.addPoint(end.row, end.col + 1, index) horiz = True # If last line was horizontal, choose the ending edge index for the incoming edge if horiz: index = self.findVertEdgeIndex(vert_edges, end.col + 1, end.row, end.row) edge.points[len(edge.points) - 1][2] = index return edge def renderFunction(self, func): # Create render nodes self.blocks = {} for block in func.blocks.values(): self.blocks[block.entry] = DisassemblerBlock(block) self.prepareGraphNode(self.blocks[block.entry]) # Populate incoming lists for block in self.blocks.values(): for edge in block.block.exits: self.blocks[edge].incoming += [block.block.entry] # Construct acyclic graph where each node is used as an edge exactly once block = func.blocks[func.entry] visited = [func.entry] queue = [self.blocks[func.entry]] changed = True while changed: changed = False # First pick nodes that have single entry points while len(queue) > 0: block = queue.pop() for edge in block.block.exits: if edge in visited: continue # If node has no more unseen incoming edges, add it to the graph layout now if len(self.blocks[edge].incoming) == 1: self.blocks[edge].incoming.remove(block.block.entry) block.new_exits += [edge] queue += [self.blocks[edge]] visited += [edge] changed = True # No more nodes satisfy constraints, pick a node to continue constructing the graph best = None for block in self.blocks.values(): if not block.block.entry in visited: continue for edge in block.block.exits: if edge in visited: continue if (best == None) or (len(self.blocks[edge].incoming) < best_edges) or ((len(self.blocks[edge].incoming) == best_edges) and (edge < best)): best = edge best_edges = len(self.blocks[edge].incoming) best_parent = block if best != None: self.blocks[best].incoming.remove(best_parent.block.entry) best_parent.new_exits += [best] visited += [best] changed = True # Compute graph layout from bottom up self.computeGraphLayout(self.blocks[func.entry]) # Prepare edge routing horiz_edges = [None] * (self.blocks[func.entry].row_count + 1) vert_edges = [None] * (self.blocks[func.entry].row_count + 1) edge_valid = [None] * (self.blocks[func.entry].row_count + 1) for row in range(0, self.blocks[func.entry].row_count + 1): horiz_edges[row] = [None] * (self.blocks[func.entry].col_count + 1) vert_edges[row] = [None] * (self.blocks[func.entry].col_count + 1) edge_valid[row] = [True] * (self.blocks[func.entry].col_count + 1) for col in range(0, self.blocks[func.entry].col_count + 1): horiz_edges[row][col] = [] vert_edges[row][col] = [] for block in self.blocks.values(): edge_valid[block.row][block.col + 1] = False # Perform edge routing for block in self.blocks.values(): start = block for edge in block.block.exits: end = self.blocks[edge] color = Qt.black if edge == block.block.true_path: color = QColor(0, 144, 0) elif edge == block.block.false_path: color = QColor(144, 0, 0) start.edges += [self.routeEdge(horiz_edges, vert_edges, edge_valid, start, end, color)] # Compute edge counts for each row and column col_edge_count = [0] * (self.blocks[func.entry].col_count + 1) row_edge_count = [0] * (self.blocks[func.entry].row_count + 1) for row in range(0, self.blocks[func.entry].row_count + 1): for col in range(0, self.blocks[func.entry].col_count + 1): if len(horiz_edges[row][col]) > row_edge_count[row]: row_edge_count[row] = len(horiz_edges[row][col]) if len(vert_edges[row][col]) > col_edge_count[col]: col_edge_count[col] = len(vert_edges[row][col]) # Compute row and column sizes col_width = [0] * (self.blocks[func.entry].col_count + 1) row_height = [0] * (self.blocks[func.entry].row_count + 1) for block in self.blocks.values(): if (int(block.width / 2)) > col_width[block.col]: col_width[block.col] = int(block.width / 2) if (int(block.width / 2)) > col_width[block.col + 1]: col_width[block.col + 1] = int(block.width / 2) if int(block.height) > row_height[block.row]: row_height[block.row] = int(block.height) # Compute row and column positions col_x = [0] * self.blocks[func.entry].col_count row_y = [0] * self.blocks[func.entry].row_count self.col_edge_x = [0] * (self.blocks[func.entry].col_count + 1) self.row_edge_y = [0] * (self.blocks[func.entry].row_count + 1) x = 16 for i in range(0, self.blocks[func.entry].col_count): self.col_edge_x[i] = x x += 8 * col_edge_count[i] col_x[i] = x x += col_width[i] y = 16 for i in range(0, self.blocks[func.entry].row_count): self.row_edge_y[i] = y y += 8 * row_edge_count[i] row_y[i] = y y += row_height[i] self.col_edge_x[self.blocks[func.entry].col_count] = x self.row_edge_y[self.blocks[func.entry].row_count] = y self.width = x + 16 + (8 * col_edge_count[self.blocks[func.entry].col_count]) self.height = y + 16 + (8 * row_edge_count[self.blocks[func.entry].row_count]) # Compute node positions for block in self.blocks.values(): block.x = int((col_x[block.col] + col_width[block.col] + 4 * col_edge_count[block.col + 1]) - (block.width / 2)) if (block.x + block.width) > (col_x[block.col] + col_width[block.col] + col_width[block.col + 1] + 8 * col_edge_count[block.col + 1]): block.x = int((col_x[block.col] + col_width[block.col] + col_width[block.col + 1] + 8 * col_edge_count[block.col + 1]) - block.width) block.y = row_y[block.row] # Precompute coordinates for edges for block in self.blocks.values(): for edge in block.edges: start = edge.points[0] start_row = start[0] start_col = start[1] last_index = edge.start_index last_pt = QPoint(self.col_edge_x[start_col] + (8 * last_index) + 4, block.y + block.height + 4 - (2 * self.charWidth)) pts = [last_pt] for i in range(0, len(edge.points)): end = edge.points[i] end_row = end[0] end_col = end[1] last_index = end[2] if start_col == end_col: new_pt = QPoint(last_pt.x(), self.row_edge_y[end_row] + (8 * last_index) + 4) else: new_pt = QPoint(self.col_edge_x[end_col] + (8 * last_index) + 4, last_pt.y()) pts += [new_pt] last_pt = new_pt start_col = end_col new_pt = QPoint(last_pt.x(), edge.dest.y + self.charWidth - 1) pts += [new_pt] edge.polyline = pts pts = [QPoint(new_pt.x() - 3, new_pt.y() - 6), QPoint(new_pt.x() + 3, new_pt.y() - 6), new_pt] edge.arrow = pts # Adjust scroll bars for new size areaSize = self.viewport().size() self.adjustSize(areaSize.width(), areaSize.height()) if self.desired_pos: # There was a position saved, navigate to it self.horizontalScrollBar().setValue(self.desired_pos[0]) self.verticalScrollBar().setValue(self.desired_pos[1]) elif self.cur_instr != None: self.show_cur_instr() else: # Ensure start node is visible start_x = self.blocks[func.entry].x + self.renderXOfs + int(self.blocks[func.entry].width / 2) self.horizontalScrollBar().setValue(start_x - int(areaSize.width() / 2)) self.verticalScrollBar().setValue(0) self.update_id = func.update_id self.ready = True self.viewport().update(0, 0, areaSize.width(), areaSize.height()) def updateTimerEvent(self): status = self.analysis.status if status != self.status: self.status = status self.statusUpdated.emit(self) if self.function is None: return if self.ready: # Check for updated code self.analysis.lock.acquire() if self.update_id != self.analysis.functions[self.function].update_id: self.renderFunction(self.analysis.functions[self.function]) self.analysis.lock.release() return # View not up to date, check to see if active function is ready self.analysis.lock.acquire() if self.analysis.functions.has_key(self.function): if self.analysis.functions[self.function].ready: # Active function now ready, generate graph self.renderFunction(self.analysis.functions[self.function]) self.analysis.lock.release() def show_cur_instr(self): for block in self.blocks.values(): row = len(block.block.header_text.lines) for instr in block.block.instrs: if self.cur_instr == instr.addr: x = block.x + int(block.width / 2) y = block.y + (2 * self.charWidth) + int((row + 0.5) * self.charHeight) self.horizontalScrollBar().setValue(x + self.renderXOfs - int(self.horizontalScrollBar().pageStep() / 2)) self.verticalScrollBar().setValue(y + self.renderYOfs - int(self.verticalScrollBar().pageStep() / 2)) return row += len(instr.text.lines) def navigate(self, addr): # Check to see if address is within current function for block in self.blocks.values(): row = len(block.block.header_text.lines) for instr in block.block.instrs: if (addr >= instr.addr) and (addr < (instr.addr + len(instr.opcode))): self.cur_instr = instr.addr self.show_cur_instr() self.viewport().update() return True row += len(instr.text.lines) # Check other functions for this address func, instr = self.analysis.find_instr(addr) if func != None: self.function = func self.cur_instr = instr self.highlight_token = None self.ready = False self.desired_pos = None self.viewport().update() return True return False def make_proc(self, addr): # Create a procedure at the requested address if one does not already exist if self.data.architecture() is None: # Architecture not defined yet, ask the user and set it now arch_dlg = ArchitectureDialog(self) if arch_dlg.exec_() == QDialog.Rejected: return False self.data.default_arch = arch_dlg.result self.analysis.lock.acquire() if addr not in self.analysis.functions: self.analysis.queue.append(addr) self.analysis.lock.release() self.function = addr self.cur_instr = None self.highlight_token = None self.ready = False self.desired_pos = None self.viewport().update() return True def navigate_to_history_entry(self, entry): self.function = entry.function self.ready = False self.desired_pos = [entry.scroll_x, entry.scroll_y] self.cur_instr = entry.cur_instr self.highlight_token = entry.highlight_token self.viewport().update() def get_history_entry(self): return DisassemblerHistoryEntry(self) def fontChanged(self): self.initFont() if self.ready: # Rerender function to update layout self.analysis.lock.acquire() self.renderFunction(self.analysis.functions[self.function]) self.analysis.lock.release() def getPriority(data, ext): if Analysis.isPreferredForFile(data): return 80 return 0 getPriority = staticmethod(getPriority) def getViewName(): return "Disassembler" getViewName = staticmethod(getViewName) def getShortViewName(): return "Disassembler" getShortViewName = staticmethod(getShortViewName) def handlesNavigationType(name): return (name == "disassembler") or (name == "make_proc") handlesNavigationType = staticmethod(handlesNavigationType)
init_plot(i11[n]) i11[n].fits[j].plot_select('l', 'k', lw, -41.1981, -35.3895) i11[n].fits[j].plot_select('l', 'k', lw, -35.3895, -31.6231, '--') i11[n].fits[j].plot_select('r', 'r', lw, -41.1981, -35.3895, '--') i11[n].fits[j].plot_select('r', 'r', lw, -35.3895, -31.6231) set_axis(i11[n], -41.1981, -31.6231) plt.xticks([-40, -35]) for k in se11[n][np.abs(se11[n][:, 2] - i11[n].fits[j].delta_phi) < 0.01]: plt.axvline(k[0], color='k', lw=1) i11 = Analysis.Run(np.load('2011_Raw.npy'), np.load('2011_Prune_strong_flat.npy'), Analysis.bval2011, np.load('2011bf_new_prune_flat_edit.npy')).prune_data i14 = Analysis.Run(np.load('2014_Raw.npy'), np.load('2014_Prune_strong.npy'), Analysis.bval2014, np.load('2014bf_new_prune_edit.npy')).prune_data soft11 = Analysis.Run(np.load('2011_Raw.npy'), np.load('2011_Prune_new_flat.npy'), Analysis.bval2011, np.load('2011bf_new_prune_flat_edit.npy')).prune_data soft14 = Analysis.Run(np.load('2014_Raw.npy'), np.load('2014_Prune_new.npy'), Analysis.bval2014, np.load('2014bf_new_prune_edit.npy')).prune_data se11 = np.load('2011se_flat_edit.npy') se14 = np.load('2014se_edit.npy') d140 = np.load('2014_0.npy') d144 = np.load('2014_4.npy')
def askopenfile(self): filename = tkinter.filedialog.askopenfilename(**self.file_opt) Analysis.path = str(filename) Analysis.main() exit(0)
D = loadmat(filename, squeeze_me=True, struct_as_record=True) xld = D["xl"] yld = D["yl"] xrd = D["xr"] yrd = D["yr"] varl = D["varxl"] varr = D["varxr"] del D ################################ Filter the Data ###################### # Get rid of anything where the width of the fit is too far # outside of norms varllo, varlhi = Analysis.middle_percent(abs(varl), 0.02) varrlo, varrhi = Analysis.middle_percent(abs(varr), 0.02) print "Variance tolerance left:", varllo, varlhi print "Variance tolerance right:", varrlo, varrhi var_sel = (varllo <= abs(varl)) * (abs(varl) <= varlhi) * (varrlo <= abs(varr)) * (abs(varr) <= varrhi) stepper = rand(len(xld)) < frac if opts.trim_edge: # Trim the outside border (where points are less reliable) xlo, xhi = Analysis.middle_percent(xld, 0.02) ylo, yhi = Analysis.middle_percent(yld, 0.02) selA = (xlo < xld) * (xld < xhi) * (ylo < yld) * (yld < yhi) # Get rid of anything where the match isn't actually that good. xdiff = median(xld - xrd) ydiff = median(yld - yrd)
midfield = fifaCleaner.convertHeightAndWeight(midfield) defenders = fifaCleaner.ConvertMonetaryValue(defenders) midfield = fifaCleaner.ConvertMonetaryValue(midfield) defenders = defenders.fillna(defenders) midfield = midfield.fillna(midfield) missing_Defender_Data = util.Missing(defenders) missing_midfielder_Data = util.Missing(midfield) util.SideSide(missing_Defender_Data, missing_midfielder_Data) print('Missing Data') X_train, X_test, y_train, y_test = dc.Spliting(defenders, 'BP') RF_Model = RandomForestClassifier(max_features='sqrt', max_leaf_nodes=5) analysis.ApplyModel(X_train, y_train, RF_Model) for position in arrayOfPositions: players = fifaCleaner.getAllPlayersInPosition(position, fifaData) playerDataFrame = pd.DataFrame(players) print(playerDataFrame.info) analysis.CorrelationMatrix('Overall', playerDataFrame) index = arrayOfPositions.index(position) if index == 0: fileName = 'GoalKeepers' elif index == 1: fileName = 'Defenders' elif index == 2: fileName = 'Midfielders' elif index == 3:
def main(self, option): overall_bestfit = [] print("Please enter the maximum number of runs:") maxruns = int(input()) print("Please enter desired number of generations: ") gens = int(input()) print("Enter Population size") self.population_size = int(input()) run_fitnesses = [] best_ind = [] for run in range(int(maxruns)): # print("Run Number %i" % run) generation = self.createPopulation(self.population_size, self.len_of_ind) print(generation) run_fitnesses.clear() for gen in range(int(gens)): print( "---------------------------------------------------------------" ) #Genetic Algorithm part x1, x2, x3 = self.extract_genes(generation) fit_value = fit.calc_fitness_value(x1, x2, x3) run_fitnesses.append(fit_value) fitness_values = fit.generation_fitness_func(generation) if option == "tournament": generation = select.tournament_select( generation, self.population_size) if option == "linear": generation = select.linear_rank_select( generation, fitness_values, self.population_size) if option == 'proportionate': generation = select.proportionate_selection( generation, fitness_values, self.population_size) generation = Crossover.random_crossover( generation, self.len_of_ind, self.population_size) generation = self.mutation(generation, self.pm) bor, vector = analyze.best_of_run(generation, run_fitnesses) self.best_of_runs += [bor] self.best_individuals += vector print("Best of run is %i" % bor) if self.exec_start == 0: overall_bestfit = bor self.exec_start = 1 if bor < overall_bestfit: overall_bestfit = bor print("Best fitness over all runs is %i" % overall_bestfit) avg = analyze.mean(run_fitnesses) print("Average of all runs is %f" % avg) print("Best of run values:") print(self.best_of_runs) print("Standard deviation from {0} independent runs is {1}".format( maxruns, np.std(self.best_of_runs))) print("Best individuals: ") for each in range(len(self.best_individuals)): print(self.best_individuals[each])
class DisplayApp: # init function def __init__(self, width, height): #motion ratio self.kn = 0.95 self.Delta = np.matrix([0, 0, 0]) self.offset = np.matrix([20,20]) self.windowOffset = np.matrix([40,40]) self.check = .001 self.dxMin = 0.5 self.dxMax = 10 self.dxDefault = 3 #store figure self.fig = None # create a tk object, which is the root window self.root = tk.Tk() # width and height of the window self.initDx = width self.initDy = height # set up the geometry for the window self.root.geometry( "%dx%d+50+30" % (self.initDx, self.initDy) ) # set the title of the window self.root.title("D.I.V.A") # set the maximum size of the window for resizing self.root.maxsize( 1024, 768 ) # bring the window to the front self.root.lift() # setup the menus self.buildMenus() # build the controls self.buildControls() #build point locator self.buildLocation() # build the objects on the Canvas self.buildCanvas() # set up the key bindings self.setBindings() # set up the application state self.objects = [] self.data = tuple() self.dataObject = None #@add to build axes function #create a view object self.viewObj = View() self.viewObj.setScreen(screen = (np.matrix([width, height])-self.windowOffset)) #print self.viewObj.getScreen() #axes labels get initialized self.xAxisLabel = StringVar() self.xAxisLabel.set("x") self.yAxisLabel = StringVar() self.yAxisLabel.set("y") self.zAxisLabel = StringVar() self.zAxisLabel.set("z") self.xlabel = None self.ylabel = None self.zlabel = None #initialize axes self.lines = [] #picture is displayed first #self.buildAxes() #self.buildLabels() self.headerData = {} #Initialize values for the plotting functions self.size = None self.color = None self.normmatrix = None self.Analysis = None self.data_num = None #self.image = ImageTk.PhotoImage(Image.open('Diva.jpg')) self.image = Tkinter.PhotoImage(file = "diva.gif") self.imageWidget = self.canvas.create_image(width/2,height/2,image=self.image) #canvas.createLIne # this method will test to see if the header given is a numeric header or not # returns true if the header is a numeric dimension and false if it is not def testHeader(self,header): if(self.dataObject == None): tkMessageBox.showwarning("Bad input","No data loaded") return False return self.dataObject.testHeader(header) #this method will test to see if there is any data loaded. # this returns true if there is data loaded, false otherwise def dataTest(self): if( self.dataObject == None): return False else: return True # this method will return a list of the headers def getHeaders(self): return self.dataObject.header_num() # This method will build the bottom section of the display that will display the location in the graph of the point clicked on # by the user when they use the control-mouse click option. def buildLocation(self): self.infoFrame = tk.Frame(self.root) self.infoFrame.pack(side = tk.BOTTOM, padx=2,pady=2,fill=tk.X) #make a separator line sep = tk.Frame( self.root, height = 2, width = self.initDx, bd = 1, relief = tk.SUNKEN) sep.pack(side=tk.BOTTOM, padx=2, pady = 2, fill = tk.X) #create a string variable to access later when we the user clicks on a data self.locationVar = tk.StringVar() self.locationVar.set('Point') tk.Label(self.infoFrame, textvariable = self.locationVar).pack() return # This will create the axes on the canvas, based on the data stored in thew ViewObj def buildAxes(self): self.vtm = self.viewObj.build() # point location matrix self.points = np.matrix([ [ 0,1,0,0,0,0], [ 0,0,0,1,0,0], [ 0,0,0,0,0,1], [ 1,1,1,1,1,1]]) self.axes = self.vtm*self.points self.lines = [] #draw axes # x-axis self.lines.append(self.canvas.create_line(self.axes[0,0],self.axes[1,0],self.axes[0,1],self.axes[1,1], fill="red")) # y-axis self.lines.append(self.canvas.create_line(self.axes[0,2],self.axes[1,2],self.axes[0,3],self.axes[1,3], fill="blue")) # z-axis self.lines.append(self.canvas.create_line(self.axes[0,4],self.axes[1,4],self.axes[0,5],self.axes[1,5], fill="yellow")) self.canvas.pack() def buildLabels(self): if( self.xlabel != None): #delete old labels self.canvas.delete(self.xlabel) self.canvas.delete(self.ylabel) self.canvas.delete(self.zlabel) #draw labels self.labels = np.matrix([ [ 0.5, 1, 0, 0, 0, 0], [ 0 , 0, 0.5, 1, 0, 0], [ 0 , 0, 0, 0, 0.5, 1], [ 1 , 1, 1, 1, 1, 1]]) offset = self.viewObj.getOffset() #bring the x axis label back to about halfway between the bottom axes and the end # of the plot #LabelTranslate = self.viewObj.translate(-offset[0,0], 0, 0) #xlabels = LabelTranslate*self.vtm*self.labels self.vtm = self.viewObj.build() labels = self.vtm*self.labels self.xlabel = self.canvas.create_text(labels[0,0] ,labels[1,0] + offset[0,1]/2 , text = self.xAxisLabel.get()) self.ylabel = self.canvas.create_text(labels[0,2]-offset[0,0]/2,labels[1,2] + offset[0,1]/2 , text = self.yAxisLabel.get()) self.zlabel = self.canvas.create_text(labels[0,4]-offset[0,0]/2,labels[1,4] + offset[0,1]/2 , text = self.zAxisLabel.get()) self.canvas.pack() # this will update the positions of the axes based on the data in the viewObj (i.e. creates a VTM) def updateAxes(self): # if the axes have not been created yet (picture is still there), do nothing if( len(self.lines) == 0): return #build the vtm self.vtm = self.viewObj.build() #multiply the axis endpoints by the vtm self.axes = self.vtm*self.points # for each line object #update the coordinates of the object self.canvas.coords(self.lines[0],self.axes[0,0],self.axes[1,0],self.axes[0,1],self.axes[1,1] ) self.canvas.coords(self.lines[1],self.axes[0,2],self.axes[1,2],self.axes[0,3],self.axes[1,3] ) self.canvas.coords(self.lines[2],self.axes[0,4],self.axes[1,4],self.axes[0,5],self.axes[1,5] ) def updateLabels(self): #if the labels have not yet been created (picture is still there), do nothing if( self.xlabel == None): return offset = self.viewObj.getOffset() self.vtm = self.viewObj.build() labels = self.vtm*self.labels self.canvas.coords(self.xlabel, labels[0,0] , labels[1,0]+ offset[0,1]/2) self.canvas.coords(self.ylabel, labels[0,2]-offset[0,0]/2, labels[1,2]+ offset[0,1]/2) self.canvas.coords(self.zlabel, labels[0,4]-offset[0,0]/2, labels[1,4]+ offset[0,1]/2) # this method builds the menu items that will be accessible for the user when the display is open def buildMenus(self): # create a new menu self.menu = tk.Menu(self.root) # set the root menu to our new menu self.root.config(menu = self.menu) # create a variable to hold the individual menus self.menulist = [] # create a file menu filemenu = tk.Menu( self.menu ) self.menu.add_cascade( label = "File", menu = filemenu ) self.menulist.append(filemenu) # create another menu for kicks cmdmenu = tk.Menu( self.menu ) self.menu.add_cascade( label = "Command", menu = cmdmenu ) self.menulist.append(cmdmenu) # menu text for the elements menutext = [ [ 'Open...', '-', 'Quit \xE2\x8C\x98-Q' ], [ 'Cluster \xE2\x8C\x98-C', 'Clean Data', 'PCA', 'Filter Data', 'Count Occurances', 'Print Data' ] ] # menu callback functions menucmd = [ [self.handleOpen, None, self.handleQuit], [self.handleCmd1, self.handleCmd2, self.handleCmdButton4, self.filterData,self.countData, self.printData] ] # build the menu elements and callbacks for i in range( len( self.menulist ) ): for j in range( len( menutext[i]) ): if menutext[i][j] != '-': self.menulist[i].add_command( label = menutext[i][j], command=menucmd[i][j] ) else: self.menulist[i].add_separator() # create the canvas object def buildCanvas(self): # this makes the canvas the same size as the window, but it could be smaller self.canvas = tk.Canvas( self.root, width=self.initDx, height=self.initDy ) #expand it to the size of the window and fill self.canvas.pack( expand=tk.YES, fill=tk.BOTH) return # build a frame and put controls in it def buildControls(self): # make a control frame self.cntlframe = tk.Frame(self.root) self.cntlframe.pack(side=tk.RIGHT, padx=2, pady=2, fill=tk.Y) # make a separator line sep = tk.Frame( self.root, height=self.initDy, width=2, bd=1, relief=tk.SUNKEN ) sep.pack( side=tk.RIGHT, padx = 2, pady = 2, fill=tk.Y) # make a cmd 1 button in the frame self.buttons = [] #width should be in characters. stored in a touple with the first one being a tag # this button will open a display for the user to plot the data self.buttons.append( ( 'cmd3', tk.Button( self.cntlframe, text="Plot", command=self.handleCmdButton3, width =5) ) ) self.buttons[-1][1].pack(side = tk.TOP) # this will reset the graph image self.buttons.append( ( 'cmd1', tk.Button( self.cntlframe, text="Reset", command=self.handleCmdButton, width=5 ) ) ) self.buttons[-1][1].pack(side=tk.TOP) # default side is top # this will display the graph in a 3d type prespective self.buttons.append( ( 'cmd2', tk.Button( self.cntlframe, text="3D", command=self.handleCmdButton2, width=5 ) ) ) self.buttons[-1][1].pack(side=tk.TOP) # default side is top #this will do the data pca self.buttons.append( ( 'cmd4', tk.Button( self.cntlframe, text="PCA", command=self.handleCmdButton4, width=5) ) ) self.buttons[-1][1].pack(side=tk.TOP) #this will bring up a popup dialog to filter the data self.buttons.append( ( 'cmd5', tk.Button( self.cntlframe, text="Filter", command=self.filterData, width=5) ) ) self.buttons[-1][1].pack(side=tk.TOP) #this will put a button on the window that will open up another dialog which will #allow the user to count the occurances of something happening given a certain condition self.buttons.append( ('comd7', tk.Button( self.cntlframe, text="Count", command = self.countData))) self.buttons[-1][1].pack(side=tk.TOP) #this will print the data to an html file self.buttons.append( ('cmd6', tk.Button( self.cntlframe, text = "Stats", command = self.printData, width = 5))) self.buttons[-1][1].pack(side=tk.TOP) #this adds a label which will show you the number of points in the data set tk.Label(self.cntlframe, text = "Points:").pack() self.Points = StringVar() self.Points.set("-") tk.Label(self.cntlframe, textvariable = self.Points).pack() #this adds the number of variables so that the user can view it tk.Label(self.cntlframe, text = "Variables:").pack() self.Variables = StringVar() self.Variables.set("-") tk.Label(self.cntlframe, textvariable = self.Variables).pack() return # This method will update the "number of points" stringVar # which displays the number of points in the current data set( as in filtered) on the # right of the window for the user def upDatePoints(self): numPoints = self.dataObject.size() self.Points.set(str(numPoints)) return # This method will update the "number of variables" stringVar # which displays the number of variables on the right of the window def upDateVariables(self): variables = self.dataObject.dim() self.Variables.set(str(variables)) return #This button will print out the data of the current data set to an #html file and then open it def printData(self): #check to see if the data is loaded test = self.checkData() if( test == False): return filename = StringVar() filename.set("stats.html") temp = getInput(parent = self.root, inputText = "filename:", entryVar = filename, title = "Choose filename to save data") #if the user did not input any filename if( filename.get() == ""): return check = self.dataObject.printDataHtml(filename.get(), filterDataBoolean = False, countStats = True) #check to see if the data was saved if( check == 1): #open the html file in a browser os.system("open " + filename.get()) # This will set all the bindings for the different mouse/button clicks def setBindings(self): self.canvas.bind( '<Button-1>' , self.handleButton1 ) self.canvas.bind( '<Button-2>' , self.handleButton2 ) self.canvas.bind( '<Button-3>' , self.handleButton3 ) self.canvas.bind( '<B1-Motion>', self.handleButton1Motion ) self.canvas.bind( '<ButtonRelease-1>', self.handleButton1Release) self.canvas.bind( '<B2-Motion>', self.handleButton2Motion ) self.canvas.bind( '<B3-Motion>', self.handleButton3Motion ) self.root.bind( '<Command-q>', self.handleModQ ) self.root.bind( '<Command-o>', self.handleModO ) self.root.bind( '<Command-t>' , self.countData) self.root.bind( '<Control-q>', self.handleQuit ) self.root.bind( '<Control-c>', self.handleCmd1 ) self.root.bind( '<Control-f>', self.filterData ) self.canvas.bind( '<Configure>', self.resize) self.canvas.bind( '<Control-Button-1>', self.handleButton4) self.canvas.bind( '<Command-Button-1>', self.handleButton5) # This method will build the pca data graph based on the information from the pca analaysis (called when you click on the # pca button) def buildPCA(self): #reset size and color dim which are not used in this plot section self.size = None self.Color = None # delete the axes for i in range(len(self.objects)): self.canvas.delete(self.objects[i]) self.objects = [] #get x,y,z data points x = (self.pcadataHeader.get('x')) y = (self.pcadataHeader.get('y')) z = self.pcadataHeader.get('z') data1 = self.Analysis.select([x,y]) #length of data leng = self.Analysis.size() #check to see if there is z data if( z == None): zData = np.zeros([leng, 1], float) data1 = np.hstack((data1,zData)) else: #obtain z data from the dataObject zData = self.Analysis.select([z]) data1 = np.hstack( (data1,zData) ) #normalize matrix build max_values = [] min_values = [] max_values.append( data1[:,0].max()) min_values.append( data1[:,0].min()) max_values.append( data1[:,1].max()) min_values.append( data1[:,1].min()) max_values.append( data1[:,2].max()) min_values.append( data1[:,2].min()) if( z == None): scale = self.viewObj.scale(1/(max_values[0]-min_values[0]), 1/(max_values[1]-min_values[1]), 0) else: scale = self.viewObj.scale(1/(max_values[0]-min_values[0]), 1/(max_values[1]-min_values[1]), 1/(max_values[2]-min_values[2])) translate = self.viewObj.translate(-min_values[0], -min_values[1], -min_values[2]) self.normmatrix = scale*translate #add a row of 1 to every data point hom = np.zeros([leng,1],float) #creates a column of just 1s hom = hom + 1; data1 = np.hstack((data1,hom)); self.data_num = data1 #build vtm vtm = self.viewObj.build() #plot the data on the graph. rowNum = 0 for row in self.data_num: point = vtm * self.normmatrix * row.T #standard size dx = self.dxDefault #standard color rgb = "#%02x%02x%02x" % (0, 0, 0) #create and add oval oval = self.canvas.create_oval( point[0,0]-dx, point[1,0]-dx, point[0,0]+dx, point[1,0]+dx, fill = rgb, outline = '') self.objects.append(oval) rowNum += 1 def clearPoints(self): # delete the axes for i in range(len(self.objects)): self.canvas.delete(self.objects[i]) self.objects = [] #reset axes labels self.xAxisLabel.set('x') self.yAxisLabel.set('y') self.zAxisLabel.set('z') self.buildLabels() # Puts points on the canvas def buildPoints(self): # delete the axes for i in range(len(self.objects)): self.canvas.delete(self.objects[i]) self.objects = [] self.size = None self.Color = None self.data_num = None #get the x,y,z,color,size header names if they exist x = self.headerData.get('x') y = self.headerData.get('y') z = self.headerData.get('z') color = self.headerData.get('color') sizeV = self.headerData.get('size') #if there is only one input, then make a histogram if( x != '' ): if( y == ''): pylab.ion() pylab.cla() #print self.dataObject.select([x]) pylab.hist(self.dataObject.select([x])) pylab.xlabel(x) pylab.ylabel("Occurances") pylab.show() return #get data data1 = self.dataObject.select([x,y]) #if the data is empty exit this method if( np.shape(data1)[1] == 0): print "No Data, remove filters or change file!" return #length of data leng = self.dataObject.size() #check to see if there is z data if( z == None): zData = np.zeros([leng, 1], float) data1 = np.hstack((data1,zData)) else: #obtain z data from the dataObject zData = self.dataObject.select([z]) data1 = np.hstack( (data1,zData) ) #add a row of 1 to every data point hom = np.zeros([leng,1],float) #creates a column of just 1s hom = hom + 1; data1 = np.hstack((data1,hom)); self.data_num = data1 #check to see if there is a color data if( color != None): colorData = self.dataObject.select([color]) #add homogeneous coordinate colorData = np.hstack((colorData,hom)) self.color = colorData else: self.color = None #check to see if there is a size data if( sizeV != None): sizeData = self.dataObject.select([sizeV]) #add homogeneous coordinate sizeData = np.hstack((sizeData,hom)) self.size = sizeData else: self.size = None #the code below is for data normalization using the (x-xMin)/(xMax-xMin) process for each #dimension of code #xy ranges xRange = self.dataObject.range_num(self.dataObject.getIndex(x)) yRange = self.dataObject.range_num(self.dataObject.getIndex(y)) # z range which is optional if( z != None): zRange = self.dataObject.range_num(self.dataObject.getIndex(z)) #scale matrix to divide by (xMax-xMin) scale = self.viewObj.scale(1/(xRange[0]-xRange[1]), 1/(yRange[0]-yRange[1]), 1/(zRange[0]-zRange[1])) #no z, so it is all zeros and can be just left unnormalized else: zRange = [0,0] #scale matrix to divide by (xMax-xMin) scale = self.viewObj.scale(1/(xRange[0]-xRange[1]), 1/(yRange[0]-yRange[1]), 0) #create the translation matrix to do the x-xMin part translate = self.viewObj.translate(-xRange[1], -yRange[1], -zRange[1]) #create the matrix that will do the normalization of the data (i.e. (x - xMin)/(xMax-xMin)) self.normmatrix = scale*translate #if there is a color category if( color != None): # normalize the color scale information colorMinMax = self.dataObject.range_num(self.dataObject.getIndex(color)) self.colorNorm = (self.color - colorMinMax[1])/(colorMinMax[0]-colorMinMax[1]) # if there is a size section, normalize it if( sizeV != None): sizeMinMax = self.dataObject.range_num(self.dataObject.getIndex(sizeV)) self.sizeNorm = (self.size - sizeMinMax[1])/(sizeMinMax[0] - sizeMinMax[1]) #build the vtm vtm = self.viewObj.build() #plot the data on the graph. rowNum = 0 #for each data vector we need to normalize it and then apply the vtm to get the screen coordinates for row in self.data_num: point = vtm * self.normmatrix * row.T #handles the size dimension if( sizeV != None): #somewhere between min and max, there is a slight cutoff at the min just so points are visible dx = self.sizeNorm[rowNum,0]*(self.dxMax) if( dx < self.dxMin): dx = self.dxMin #set the default size if no size dim else: dx = self.dxDefault #handles the color dimension if( color != None): #get the type of the data (should be either numeric or enum) type = self.dataObject.getType(self.dataObject.getIndex(color)) #since it is normalized between 0,1 this is a direct correlation alpha = self.colorNorm[rowNum,0] #base it off of the HSV color scale if(type == 'enum'): #multiply alpha by 0.8 to prevent going all the way around rgb = colorsys.hsv_to_rgb(alpha*0.8, 1, 1) rgb = (int(float(rgb[0])*200), int(float(rgb[1])*200), int(float(rgb[2])*200)) rgb = "#%02x%02x%02x" % (rgb[0], rgb[1], rgb[2]) else: rgb = "#%02x%02x%02x" % (200*alpha, 0.0, (1-alpha)*200) #default color == black else: rgb = "#%02x%02x%02x" % (0, 0, 0) # this part stores the location of the point in data space to the oval so that it can be accessed later when we only know # the screen coordinates. if( z != None): Location = '%s: %.2f , %s: %.2f , %s: %.2f' % ( x, row[0,0], y, row[0,1], z, row[0,2] ) else: Location = '%s: %.2f , %s: %.2f' % ( x, row[0,0], y, row[0,1] ) #check to see if size and color are being used if( sizeV != None): Location += ', %s: %.2f' % (sizeV, self.size[rowNum,0]) if( color != None): Location += ', %s: %.2f' % (color, self.color[rowNum,0]) #add in the row number Location += ' %d' % (rowNum) #create and add oval oval = self.canvas.create_oval( point[0,0]-dx, point[1,0]-dx, point[0,0]+dx, point[1,0]+dx, fill = rgb, outline = '', tag = Location) self.objects.append(oval) rowNum += 1 # This method will update all of the points in the canvas graph based on the vtm in the viewObj. Color dim does not need to be updated because # it is never reset def updatePoints(self): if( len(self.objects) == 0): return # if there are data objects plotted if ( self.data_num != None): vtm = self.viewObj.build() rowNum = 0 #for each row in the numeric data for row in self.data_num: point = vtm *self.normmatrix* row.T #handles the size dimension if( self.size != None): #somewhere between min and max, there is a slight cutoff at the min just so points are visible dx = self.sizeNorm[rowNum,0]*(self.dxMax) if( dx < self.dxMin): dx = self.dxMin #set the default size if no size dim else: dx = self.dxDefault #color dim does not need to change here! #adjust positions self.canvas.coords(self.objects[rowNum],point[0,0]-dx, point[1,0]-dx, point[0,0]+dx, point[1,0]+dx) rowNum += 1 # this method will load in the data. It does not plot the data and only loads the data into the programs memory def handleOpen(self): print 'handleOpen' #open file that has the data fobj = tkFileDialog.askopenfile( parent=self.root, mode='rU', title='Choose a data file (.csv)' ) #error checking if( fobj == None): return False self.dataObject = csvDataReader(pathName = fobj.name) self.upDatePoints() self.upDateVariables() print "Data Loaded!" #this returns the axes to the default location and then builds the labels and axes self.returnDefault() if( len(self.lines) == 0): self.buildAxes() #deletes the image if it is still on the canvas self.canvas.delete(self.imageWidget) self.clearPoints() return True def handleQuit(self, event=None): print 'Terminating' self.root.destroy() def handleModQ(self, event): self.handleQuit() def handleModO(self, event): self.handleOpen() # This method tests to see if there is any data loaded # if there is no data loaded it will prompt the user to load data. # It will then return True if there is data loaded (either before or during the method) # or False if there is no data loaded def checkData(self): # no dataObject if( self.dataObject == None): print "Open data file" #create dialog box response = tkMessageBox.askyesno(title = "No Loaded Data!", message = "Load data?") #if the user wants to laod data if response: check = self.handleOpen() if(check == False): return False return True #No data loaded, return false else: return False else: return True # this method will reset the view object to the default settings # for the viewer. def returnDefault(self): self.viewObj.setExtent(np.matrix([1,1,1])) self.viewObj.setOffset(np.matrix([20,20])) #self.viewObj.setScreen(np.matrix([400,400])) self.viewObj.setScreen(np.matrix([self.canvas.winfo_width(),self.canvas.winfo_height()]) - self.windowOffset) self.viewObj.setU(u = np.matrix([1,0,0])) self.viewObj.setVPN(vpn = np.matrix([0,0,-1])) self.viewObj.setVRP(vrp = np.matrix([0.5,0.5,1])) self.viewObj.setVUP(vup = np.matrix([0,1,0])) self.updateAxes() self.updatePoints() self.updateLabels() return #reset to default def handleCmdButton(self): print 'handling command button' self.returnDefault() return # 3d view for the user def handleCmdButton2(self): test = self.checkData() if( test == False): return print 'handling command button2' self.viewObj.setExtent(np.matrix([ 1.96429238, 1.96429238, 1.96429238])) self.viewObj.setOffset(np.matrix([20,-20])) #self.viewObj.setScreen(np.matrix([400,400])) self.viewObj.setScreen(np.matrix([self.canvas.winfo_width(),self.canvas.winfo_height()])) #print self.viewObj.getScreen() self.viewObj.setU(u = np.matrix([-0.89773056, 0.10085255, -0.42884566])) self.viewObj.setVPN(vpn = np.matrix([ 0.44036241, 0.23344776, -0.86693892])) self.viewObj.setVRP(vrp = np.matrix([ 0.43337459, 0.56723291, 0.76298161])) self.viewObj.setVUP(vup = np.matrix([-0.01268006, 0.96712507, 0.25398486])) self.updateAxes() self.updateLabels() self.updatePoints() return # this method will open up the plot dialog for the user so that it can determine which categories in the data # to plot. def handleCmdButton3(self): #check to see if there is data loaded test = self.checkData() if( test == False): return print 'handling command button3' if( self.dataObject.filtersize() == 0): tkMessageBox.showwarning("No data!","Remove Filters from the data or load a different data set!") return #create a new dictionary that will store the headers for the x,y,z,size, and color dim self.headerData = {} #open input dialog inputBox = MyDialog(parent = self.root,displayClass = self, dataHeader = self.headerData) if(self.headerData['check'] == True): #plot the points self.buildPoints() #check points self.xAxisLabel.set( self.headerData.get('x') ) self.yAxisLabel.set( self.headerData.get('y') ) #z is an optional component z = self.headerData.get('z') if( z == None ): z = "" self.zAxisLabel.set( z ) self.buildLabels() #This mehtod will open up a dialog for the user so that it can determine which categories it wants to plot after # PCA analysis has been computed on the data set def handleCmdButton4(self): #check to see if data has been loaded test = self.checkData() if( test == False): return self.size = None self.color = None #check to see if there is any data in the set after filters applied if( self.dataObject.filtersize() == 0): tkMessageBox.showwarning("No data!","Remove Filters from the data or load a different data set!") return print "handling command button 4" # run pca on all of the numerical data self.Analysis = Analysis(name = "PCA", data = self.dataObject.getData_num()) #dictioanry that will store the headers for the pca analsysis plots self.pcadataHeader = {} # this will run the dialog for pca inputBox = PCADialog(parent = self.root, pcaDataHeader = self.pcadataHeader, displayClass = self, Analysis = self.Analysis) if( self.pcadataHeader['check'] == True): self.xAxisLabel.set( self.pcadataHeader.get('x') ) self.yAxisLabel.set( self.pcadataHeader.get('y') ) z = self.pcadataHeader.get('z') #z is optional if( z == None ): z = "" self.zAxisLabel.set(z) self.buildLabels() #graphs the pca points self.buildPCA() if( self.pcadataHeader.get('saveData') ): filename = self.pcadataHeader.get('filename') filename = filename + ".csv" self.Analysis.writeData(filename = filename) #This method will open up the filter choice menu dialog where the user will #be able to choose which fitlers to add/remove from the data. #this will also have an option for removing missing data. def filterData(self, event=None): #check to see if the data is loaded test = self.checkData() if( test == False): return a = filterDialog(parent = self.root, displayClass = self, dataObj = self.dataObject, title = "Filter") return # This method will open the countInfo Dialog box where the user can count the number # of data points that meet two conditions which will be saved in the data object def countData(self, event = None): #check to see if the data is loaded test = self.checkData() if( test == False): return #open dialog temp = countInfo(parent = self.root, dataObject = self.dataObject, title = "Count Occurrences within the Data:") return # this method will run the kmeans clustering algorithm on the loaded data # a popup dialog will appear with options for the user to choose which columns # to run on the kmeans clustering. Default being use all numeric columns # the user can also choose the number of clusters to have and the name # of this cluster run def handleCmd1(self, event = None): #check to see if the data is loaded test = self.checkData() if( test == False): return #check to see if there is any data if( self.dataObject.filtersize() == 0): tkMessageBox.showwarning("No data!","Remove Filters from the data or load a different data set!") return #obtain data self.clusterInfo = {} clusterDialogBox = ClusterDialog(parent = self.root, displayClass = self, clusterInfo = self.clusterInfo,title="Clusters") #if nothing is chosen do nothing and exit if( self.clusterInfo.get("headers") == None): return headers = self.clusterInfo.get("headers") self.dataObject.cluster(headers = headers, clusters = self.clusterInfo.get("clusters"), name = self.clusterInfo.get("name"), normalize = True) tkMessageBox.showwarning("Clustering Complete", "The clustering process has now completed!") print 'handling command 1' return #this method will be used to clean data def handleCmd2(self): #check to see if the data is loaded test = self.checkData() if( test == False): return #initialize filename variable filename = StringVar() temp = getInput(parent = self.root, inputText = "Choose Filename:", entryVar = filename, title = "Clean data of missing information") #if nothing is input if( filename.get() == ""): return #clean the data self.dataObject.setMissingToAverage() #save it self.dataObject.saveData(filename.get()) #load the new data back into the system self.dataOBject = csvDataReader(pathName = filename.get()) self.upDatePoints() self.upDateVariables() print "Data Loaded!" return def handleCmd3(self): print 'handling command 3' # Stores scaling data initial values def handleButton3(self, event): print "handling button3" self.baseClick3 = (event.x, event.y) self.textent = self.viewObj.getExtent() return #scales the values of the axes/data based on y movement when button 3 is depressed, all based off of initial click location def handleButton3Motion(self, event): diff = event.y-self.baseClick3[1] #determine scale scale = 1+(3.0*diff)/self.root.winfo_height() #print "scale",scale #check to see extremes of scale if( scale < 0.1): scale = 0.1 elif( scale > 3): scale = 3 #change the extent self.viewObj.setExtent(self.textent*(scale)) #update drawings self.updateAxes() self.updateLabels() self.updatePoints() # store initial value for translation def handleButton1(self, event): #print 'handle button 1: %d %d' % (event.x, event.y) self.baseClick1 = (event.x, event.y) closestObj = None #for obj in self.objects: # store initial values before rotation (i.e. clone view object) def handleButton2(self, event): #print 'handle button 2: %d %d' % (event.x, event.y) self.baseClick2 = (event.x,event.y) self.viewObjClone = self.viewObj.copy(View()) return # rotates the data based on movement from initial click location. 400 px is about 1 full rotation def handleButton2Motion(self, event): divConst = 200.0 #rotation angles delta0 = ((event.x-self.baseClick2[0])/divConst)*math.pi delta1 = ((event.y-self.baseClick2[1])/divConst)*math.pi #clone view object viewObjClone2 = self.viewObjClone.copy(View()) #rotate viewObjClone2.rotateVRC(delta0,delta1) # update points, axes, view obj self.viewObj = viewObjClone2 self.updateAxes() self.updateLabels() self.updatePoints() return # This is executed when the user releases button1 of the mouse and starts the callback # for the continual "inertial" motion def handleButton1Release(self, event): #call my callback method self.callback_Motion() # This call back method will continue to be called and apply the inertial effects of the tranlsation # caused by the user until it reaches some minimum. The constants self.kn and self.check are set in the # constructor method. This method will be called by the program every 20 ms to update the axes def callback_Motion(self): # update our travel vector by some scalar constant self.kn self.Delta = self.Delta*self.kn # this checks to see if the change is between -check < x < check for x,y,z variables if( self.Delta[0,0] < self.check and self.Delta[0,0] > -self.check): if( self.Delta[0,1] < self.check and self.Delta[0,1] > -self.check): if( self.Delta[0,2] < self.check and self.Delta[0,2] > -self.check): # if it is exit and end callbacks return # update the vrp location vrp = self.viewObj.getVRP() vrp = vrp + self.Delta self.viewObj.setVRP(vrp) #update points and axes self.updateAxes() self.updateLabels() self.updatePoints() #set callback method call self.root.after(20, self.callback_Motion) return # this method will handle the 'control-mouse click' combo which will display the point location in data space of the nearest point # to the user's click def handleButton4(self,event): #mouse click location x = event.x y = event.y #gets id of closest oval closest = self.canvas.find_closest(x, y, halo = None, start = None) # this will get the tag of the closest oval which stores the location in data space of the point tag = self.canvas.gettags(closest[0]) # this creates the pointInfo string which will be put on the display pointInfo = '' for i in range(0,len(tag)-2): pointInfo += " " + tag[i] #update display self.locationVar.set("Point: " + pointInfo) return # this method will handle the 'command-mouse click' combo which will open a display dialog which will display # all of the data for that data vector in an image def handleButton5(self,event): #mouse click location x = event.x y = event.y #gets id of closest oval closest = self.canvas.find_closest(x, y, halo = None, start = None) # this will get the tag of the closest oval which stores the location in data space of the point tag = self.canvas.gettags(closest[0]) length = len(tag) if(tag[length-1] == 'current'): rowNum = tag[length-2] else: rowNum = tag[length-1] #rowNum = tag[length-2] #print tag #print rowNum dialog = InfoBox(parent = self.root, dataObject = self.dataObject, rowNumber = rowNum, title = "Data Information") return # handles translation of the axes / data def handleButton1Motion(self, event): # this creates a differential motion information for us by updating this # every time the mouse is moved # calculate the difference diff = ( event.x - self.baseClick1[0], event.y - self.baseClick1[1] ) self.baseClick1 = (event.x, event.y) norm = [] norm.append( float(diff[0])/self.root.winfo_width()) #print "width", self.root.winfo_width() norm.append( float(diff[1])/self.root.winfo_height()) #print "norm",norm adj = 0.8 #change amount delta0 = adj*norm[0]*self.viewObj.getExtent()[0,0] delta1 = adj*norm[1]*self.viewObj.getExtent()[0,1] # adjust vrp vrp = self.viewObj.getVRP() self.Delta = delta0*self.viewObj.getU() + delta1*self.viewObj.getVUP() vrp = vrp + self.Delta self.viewObj.setVRP(vrp) #redraw points self.updateAxes() self.updateLabels() self.updatePoints() # handles if if the canvas size is changed, will reset the data def resize(self,event): #self.viewObj.setScreen(np.matrix([self.canvas.winfo_width(), self.canvas.winfo_height()])) #self.handleCmdButton() #self.updateAxes() #self.updatePoints() self.returnDefault() return def main(self): print 'Entering main loop' #lets everything just sit and listen self.root.mainloop()
def initiate_analysis(self): # Load input data from tables section = self.get_geometry() SF = self.getSF() Mat = self.get_material() # check if geometry is valid valid, msg = section.valid() if not valid: self.show_msg_box(['The defined geometry is not valid', msg]) self.Res = None self.refresh_visible_plots() return print(section) print(Mat) print(SF) # switch to Loading & Result tab self.tabWidget.setCurrentIndex(3) # set mouse cursor to WaitCursor self.setCursor(QtCore.Qt.WaitCursor) try: # Call analysis if self.checkBox_analSLS_1.isChecked(): # execute SLS analysis string = self.checkBox_analSLS_1.text() self.statusbar.showMessage(string + ' analysis initiated') self.Res = Analysis.SLS_analysis(section, SF, Mat) error_msg = None # errors are now passed as exceptions self.load_fac_label.setText('No load-factor currently applied' ) # <-- might not be needed self.statusbar.showMessage(string + ' analysis completed') elif self.checkBox_analULS_1.isChecked(): # execute ULS analysis string = self.checkBox_analULS_1.text() self.statusbar.showMessage(string + ' analysis initiated') self.Res, error_msg = Analysis.ULS_analysis(section, SF, Mat) self.statusbar.showMessage(string + ' analysis completed') else: self.Res = None error_msg = 'No analysis method is checked' self.load_fac_label.setText('') except Analysis.MyOptimizerError as e: # caught a MyOptimizerError exception self.show_msg_box([str(e), e.discription]) error_msg = None else: # no MyOptimizerError exception self.load_fac_label.setText('No load-factor currently applied') # Show message if error_msg: self.show_msg_box(error_msg, set_load_fac_label=True) else: self.load_fac_label.setText('No load-factor currently applied') # update result plot self.refresh_visible_plots() # return mouse cursor to normal ArrowCursor self.setCursor(QtCore.Qt.ArrowCursor)
def __init__(self, data, filename, view, parent): super(DisassemblerView, self).__init__(parent) self.status = "" self.view = view self.data = data for type in ExeFormats: exe = type(data) if exe.valid: self.data = exe self.view.exe = exe break # Create analysis and start it in another thread self.analysis = Analysis(self.data) self.analysis_thread = threading.Thread(None, self.analysis_thread_proc) self.analysis_thread.daemon = True self.analysis_thread.start() # Start disassembly view at the entry point of the binary if hasattr(self.data, "entry"): self.function = self.data.entry() else: self.function = None self.update_id = None self.ready = False self.desired_pos = None self.highlight_token = None self.cur_instr = None self.scroll_mode = False self.blocks = {} self.show_il = False self.simulation = None # Create timer to automatically refresh view when it needs to be updated self.updateTimer = QTimer() self.updateTimer.setInterval(100) self.updateTimer.setSingleShot(False) self.updateTimer.timeout.connect(self.updateTimerEvent) self.updateTimer.start() self.initFont() # Initialize scroll bars self.width = 0 self.height = 0 self.setHorizontalScrollBarPolicy(Qt.ScrollBarAsNeeded) self.setVerticalScrollBarPolicy(Qt.ScrollBarAsNeeded) self.horizontalScrollBar().setSingleStep(self.charWidth) self.verticalScrollBar().setSingleStep(self.charHeight) areaSize = self.viewport().size() self.adjustSize(areaSize.width(), areaSize.height()) # Setup navigation self.view.register_navigate("disassembler", self, self.navigate) self.view.register_navigate("make_proc", self, self.make_proc) self.search_regex = None self.last_search_type = FindDialog.SEARCH_HEX
def quasi_airy(center_x, center_y, width_x, width_y, offset, height): def quairy_func(x,y): R = sqrt((x - center_x)**2/width_x + (y - center_y)**2 / width_y) return offset + height * jn(1, R)/(R+1e-6) return quairy_func if __name__ == '__main__': funcs = {"Quasi-Airy": quasi_airy, "Gaussian": Analysis.gaussian, \ "Airy" : Analysis.airy, "Bessel", bessel} rmses = {} for i in funcs.keys(): rmses[i] = 0 try: os.uname() dirname = '/Users/pcombs/Documents/RotationData/2009-12-08-FITS/*' bgname = '/Users/pcombs/Documents/RotationData/2009-12-08/MED_2009-12-08.fits' except AttributeError: dirname = r'C:\Documents and Settings\admin\Desktop\Peter\2009-12-09-FITS/8*' bgname = r'C:\Documents and Settings\admin\Desktop\Peter\MED_2009-12-08.fits' filelist = glob(dirname) filelist = [os.path.join(dirname, f) for f in filelist] filelist = filelist for func_name in funcs.keys(): fit = Analysis.fit2d(xs, ys, data, fcn = funcs[func_name])
import os import Image_Process import cv2 import cv2.cv as cv import numpy as np import BoW import SVM import Analysis import Descriptors B = BoW.BoW("BoWSS.pkl") # if B.loaded: # print "BoW already computed" # exit path = "/media/iglu/Data/rgbd-dataset" i = 0 Analysis.initial() start_time = timeit.default_timer() i = 0 for x in os.listdir(path): # if i == 0: # i += 1 # continue # else: mn=0 D=False R=False if x.startswith("Test"): continue for y in os.listdir(path + "/" + x): for z in os.listdir(path +"/"+x +"/"+y+"/"): if z.endswith("_crop.png"):
import numpy as np import matplotlib.pyplot as plt import Analysis as anl """" get data from NNGT saved files_double_event """ rootpath = "/home/mallory/Documents/These/javier-avril-CR/Simulations2/GaussianNetworks/IBneurons/" filename = "2000Gaussian_30_6.0_weight166.67withminis.txt" tmin = 0. tmax = 200000. raster = anl.load_raster(rootpath + filename, tmin=tmin, tmax=tmax, with_space=False) """ Find the bursts : ce passage prend du temps, tu peux le passer si tu veux juste la phase et ses stat. """ plot_phase = True #Choose time step in ms step = 5. # Ici je change les tmin et tmax parce que la phase n'est definit que # entre deux spike, donc je regarde uniquement à ces temps la. tmin = np.nanmin(raster[:, 0]) tmax = np.nanmax(raster[:, -1])
title= f'Final predictions from ARIMA model for "{stock_name}" stock', color=["#c6e2ff", "#deaddd"], linewidth=1.5) if save_plot: plt.savefig('./demonstration_images/arima_predictions.png') plt.show() plt.figure(figsize=(12, 8)) sns.kdeplot(res["Residuals"], color="#c7c6ff", shade=True, linewidth=2) plt.title( f'Density of the residuals from the auto-regression for "{stock_name}" stock' ) plt.xlabel('Residuals') plt.ylabel('Density') if save_plot: #fig_r = ax_r.get_figure() plt.savefig('./demonstration_images/arima_residuals.png') plt.show() if __name__ == '__main__': df = Analysis.get_data('./Data/AMZN.csv') df = df[["Close"]] df.rename(columns={0: 'Close'}, inplace=True) arima_model = ARIMA(0.8, 'Close') df_c, res = arima_model.run(df, True) arima_model.plot_results(df_c, res, 'AMZN', save_plot=True)
help="") parser.add_option('-q', '--quiet', dest = "quiet", action = "store_true", default = False) parser.add_option('-o', '--output-file', dest="output", default=None) parser.add_option('-c', '--center', dest="center", default=False, help="For measuring colocalization distance, take as distance from" "the average colocalization", action="store_true") opts, args = parser.parse_args() if not opts.map: if opts.x1 and opts.x2 and opts.y1 and opts.y2: opts.xshift = opts.x2 - opts.x1 opts.yshift = opts.y2 - opts.y1 else: mapping = Analysis.loadmapping(opts.map) if opts.map2: mapping2 = Analysis.loadmapping(opts.map2) timepoints = defaultdict(list) tpmins = [0]*100 tpmaxes = [0]*100 imgname_finder = re.compile('(.*)_spot[0-9]+_[^_]+_xy(_only)?.txt') spotnum_finder = re.compile('spot([0-9]+)_') channel_finder = re.compile('spot[0-9]+_([0-9]{3})_xy(_only)?') #print map(glob, args) max_timepoint = 0
Load data, analyze, and output results. """ import os import pandas import pdb import sys import Analysis import Constants import Basketball import Misc if __name__ == "__main__": tournaments = {} analysis_options = Analysis.get_objects() num_analysis_options = len(analysis_options) # prompt user to select analysis method prompt = "Please enter number of desired analysis method:\n" for i in range(num_analysis_options): prompt += "%s) %s\n" % (i, analysis_options[i].get_name()) user_input = raw_input(prompt) # check user input if user_input.isdigit() and (int(user_input) in range(num_analysis_options)): analysis = analysis_options[int(user_input)] else: raise Exception("Invalid entry '%s', please enter one of following values %s." % (user_input, range(num_analysis_options))) # prompt user for Kaggle output kaggle_output_options = {
torch.save(net.state_dict(), "Unet2_model_" + leg + "_min_val_loss") # save model min_model_epoch = epoch + 1 disp = 'Epoch %d: Train loss: %.6f, Val loss: %.6f' % ( epoch + 1, running_loss / count, val_loss_temp) write_2_log(disp) print(disp) val_loss.append(val_loss_temp) train_loss.append(running_loss / count) running_loss = 0.0 if (epoch + 1) in model_saved: # save checkpoint model_name = "Unet2_model_" + str(epoch + 1) + "epoch_" + leg torch.save(net.state_dict(), model_name) # save model Analysis.analysis(model_file=model_name, valloader=valloader) write_2_log("Best model at epoch: " + str(min_model_epoch)) write_2_log("Total training time: " + str((time.time() - start_begining) / 3600.0)) val_loss_file = "data/results" + "/val_loss_" + str( N_epoch) + "epoch_" + leg + ".npy" train_loss_file = "data/results" + "/training_loss_" + str( N_epoch) + "epoch_" + leg + ".npy" np.save(val_loss_file, val_loss) np.save(train_loss_file, train_loss) print("total processing time in seconds: ", time.time() - start_begining) print("total processing time in mins: ", (time.time() - start_begining) / 60.0)
def polarityDashboard(origStock, stockList, compareData): ''' Make a dashboard for the news polarity stuff Output it to polarityDashboard.html ''' #relevanceBar = barPlot(compareData, "NEWS_SIMILARITY") titlePolarities, titleBias, textPolarities, textBias = Analysis.articleSentiments( stockList) allFigures = [] #allFigures.append([relevanceBar]) #para = Paragraph(text="On this dashboard, you can see the polarity of relevant news articles and headlines. Strongly positive or negative headlines will drive short-term volatility, whereas the polarity of articles impact long-term growth.") #get averages of each means = [] for oneStockTitles, oneStockTexts in zip(titlePolarities, textPolarities): if len(oneStockTitles) > 0 and len(oneStockTexts) > 0: means.append((statistics.mean(oneStockTitles) + statistics.mean(oneStockTexts)) / 2.0) if len(means) > 0: xBarNames = compareData["stock_SYMBOL"] #the things that we're listing polBar = figure(x_range=xBarNames, title="Polarity Summary", y_axis_label='Polarity', height=300, sizing_mode='scale_width') polBar.vbar(x=xBarNames, top=means, width=0.9) allFigures.append([polBar]) colors = ["blue", "green", "orange", "purple", "magenta", "cyan", "yellow"] count = 1 for oneStock, stockTitles in zip(stockList, titlePolarities): titleLen = list(range(1, len(stockTitles) + 1)) polarityFigure = figure(title=f"{oneStock.name} Headline Polarity", x_axis_label='Time', y_axis_label='Polarity', height=300, sizing_mode='scale_width') polarityFigure.line(x=titleLen, y=stockTitles, legend=oneStock.symbol, line_width=2, color=colors[count]) if len(allFigures) - 1 > 0 and len( allFigures[len(allFigures) - 1]) >= 2: allFigures.append([polarityFigure]) else: if len(allFigures) - 1 <= 0: allFigures.append([polarityFigure]) else: allFigures[len(allFigures) - 1].append(polarityFigure) count += 1 if count > 5: count = 0 for oneStock, stockTexts in zip(stockList, textPolarities): textLen = list(range(1, len(stockTexts) + 1)) polarityFigure = figure(title=f"{oneStock.name} Article Text Polarity", x_axis_label='Time', y_axis_label='Polarity', height=300, sizing_mode='scale_width') polarityFigure.line(x=textLen, y=stockTexts, legend=oneStock.symbol, line_width=2, color=colors[count]) if len(allFigures) - 1 > 0 and len( allFigures[len(allFigures) - 1]) >= 2: allFigures.append([polarityFigure]) else: if len(allFigures) - 1 <= 0: allFigures.append([polarityFigure]) else: allFigures[len(allFigures) - 1].append(polarityFigure) count += 1 if count > 6: count = 0 grid = gridplot(allFigures) return components(grid)
import TrialMoves as TM import Analysis as an if __name__=='__main__': (lattice,graft_points,chains) = init.initialize_lattice((200,200,20),200,16,[2,3],20) #previously 779 print len(graft_points) count = 0 n = sys.argv[1] alph = sys.argv[2] #n = 5 #alph = 'f' an.chains_to_xyz(chains, 'InitDual_'+str(n)+alph, lattice) for i in range(0,int(n)): rando = rnd.uniform(0,1) if rando>.66: (lattice, chains,total_energy, acc) = TM.cbmc(lattice,chains) print "CBMC" elif rando > .33: (lattice, chains, total_energy, acc) = TM.take_empty(lattice, chains, graft_points) print "Empty" else: (lattice, chains, total_energy, acc) = TM.swap(lattice, chains) print "SWAP" an.store_energies(total_energy, n, alph) # Should add matrices for chemical moeity and identity in here
dcfs.getData() # Get LA City zip codes (Web scraping) # Get Walk Score data based on LA City zip codes (Web scraping) print("\n * Getting LA City zip copdes and walk score data") walkScore = classWalkScore.ClassWalkScore(args.source) walkScore.getLAcityZip() walkScore.getData() # Get LA population data (API) print("\n * Getting LA population data") pop = classPop.ClassPop(args.source) pop.getLApop() # Get crime report data (API) print("\n * Getting crime records data") crime = classCrime.ClassCrime(args.source) crime.getData() # Cleaning data and data modeling print("\n * Clean data and modeling") cleanReferral.CleanReferral() cleanReferral.ZipCrime() # Analyze each dataset and combined dataset Analysis.loadGeoJSON() Analysis.Crime_analysis() Analysis.DCFS_analysis() Analysis.Combine_analysis() Analysis.Correlation_graph()