def OnPressEnter2(self,event):
solution = self.entryVariable2.get()
part = solution.split("-")
rw_csv.deleteOldGoal(part[1])
item = part[1].split(" ")
score = item[len(item)-1]
rw_csv.writeScore({part[0]:int(score)})
for i in self.tree.get_children():
self.tree.delete(i)
result = rw_csv.readScore()
self.tree.insert("" , 0, text="Score", values=(result["Family"],result["Travel"],result["Studying"],result["Friend"],result["Volunteer"]))
goal = rw_csv.readGoal()
id2 = self.tree.insert("", 1, "Task", text="Task")
family_text = ""
for x in goal["Family"]:
family_text = family_text + x+ "\n"
travel_text = ""
for x in goal["Travel"]:
travel_text = travel_text + x+ "\n"
studying_text = ""
for x in goal["Studying"]:
studying_text = studying_text + x+"\n"
friend_text = ""
for x in goal["Friend"]:
friend_text = friend_text + x+"\n"
volunteer_text = ""
for x in goal["Volunteer"]:
volunteer_text = volunteer_text + x+"\n"
self.tree.insert(id2, "end", text="", values=(family_text,travel_text,studying_text,friend_text,volunteer_text))
Analysis.makingBalanceLifeGraph(result["Family"],result["Travel"],result["Studying"],result["Friend"],result["Volunteer"])
img = self.img = ImageTk.PhotoImage(Image.open('BalanceLifeGraph.png'))
self.panel = Tkinter.Label(self,image=img)
self.panel.pack(side = "left")
def single_processor(filequeue, pointslist, donelist, bgimg = None,
basedir = None, thresh=THRESH):
while not filequeue.empty():
fname = filequeue.get()
frame_num = int(path.splitext(path.split(fname)[-1])[0])
if basedir:
pass
if len(donelist)%100 == 0:
sys.stderr.write("Completed %d\n"%(len(donelist)))
sys.stderr.write("Found %d holes\n" %
len(pointslist))
try:
img1, img2 = Analysis.loadsplit(fname, bgimg = bgimg)
pointslist.extend(Analysis.findpairs(img1, img2, thresh=thresh,
DEBUG=DEBUG, abs_thresh=True,
frame_num = frame_num))
donelist.append(fname)
except IOError:
pass
except KeyboardInterrupt:
return
def __init__(self,Pathx, B):
path = "/media/iglu/Data/DatasetIglu"
i = 0
Analysis.initial()
files = open("Output_1_L.txt", 'w')
start_time = timeit.default_timer()
for x in os.listdir(path):
# if i == 0:
# i += 1
# continue
# else:
if Pathx != x:
continue
for y in os.listdir(path + "/" + x) :
if (y != "Objects.txt") and not y.endswith(".zip"):
# print path+"/"+x+"/"+y+"/k1"
f = open(path + "/" + x + "/" + y + "/k1" + "/List.txt", 'r')
f2 = open(path + "/" + x + "/" + y + "/speech.txt", 'r')
speak = f2.readline()
if speak == 'This...':
sp = 1
elif speak == 'That...':
sp = 2
else:
sp = 0
mn = 0
for line in f:
Time = line
file1 = next(f).rstrip('\n')
file2 = next(f).rstrip('\n')
Label = next(f).rstrip('\n')
# files.write(RGB.__str__())
# files.write(Depth.__str__())
# files.write(Label+" "+x+" "+file1+"\n")
if mn == 0:
RGB = cv2.imread(path + "/" + x + "/" + y + "/k1" + "/RGB/" + file1)
Depth = cv2.imread(path + "/" + x + "/" + y + "/k1" + "/Depth/" + file2)
Analysis.addDepth(Depth)
Analysis.addRGB(RGB)
Analysis.addSpeech(sp)
Analysis.addScene(Label)
else:
if mn == 2:
mn = -1
mn += 1
# print "Enviado " + x + " " + y
i += 1
elapsed = timeit.default_timer() - start_time
print "Tiempo: " + elapsed.__str__()
print "Starting Training"
start_time = timeit.default_timer()
Analysis.CompleteAnalysis(B, False, True, 0.3, files)
files.close()
# Analysis.Mostrar()
# code you want to evaluate
elapsed = timeit.default_timer() - start_time
print "Tiempo: " + elapsed.__str__()
def makebackground(filelist, n = 300): import numpy as np from random import randrange as rand filedesc = Analysis.imread(filelist[0]) [r,c] = np.shape(filedesc) mat = np.empty((r,c,n)) for i in range(n): mat[:,:,i] = Analysis.imread(filelist[rand(len(filelist))]) return np.median(mat, axis=2)
def _line_spectrum(data, min, line, dE, width, width_error):
# Draw histogram
n, bins = Analysis.histogram(data, binsize=binsize)
if method in ("cs"):
gn, gbins = Analysis.group_bin(n, bins, min=min)
else:
# No grouping in mle and ls
gn, gbins = n, bins
ngn = gn/(np.diff(gbins))
ngn_sigma = np.sqrt(gn)/(np.diff(gbins))
cbins = (gbins[1:]+gbins[:-1])/2
if plotting:
figure()
if width_error is not None:
label = 'FWHM$=%.2f\pm %.2f$ eV' % (width, width_error)
else:
label = 'FWHM$=%.2f$ eV (Fixed)' % width
if method == "cs":
errorbar(cbins, ngn, yerr=ngn_sigma, xerr=np.diff(gbins)/2, capsize=0, ecolor='k', fmt=None, label=label)
else:
hist(data, bins=gbins, weights=np.ones(len(data))/binsize, histtype='step', ec='k', label=label)
E = np.linspace(bins.min(), bins.max(), 1000)
model = Analysis.normalization(ngn, gbins, dE, width, line=line, shift=shift) \
* Analysis.line_model(E, dE, width, line=line, shift=shift, full=True)
# Plot theoretical model
plot(E, model[0], 'r-')
# Plot fine structures
for m in model[1:]:
plot(E, m, 'b--')
xlabel('Energy$\quad$(eV)')
ylabel('Normalized Count$\quad$(count/eV)')
legend(frameon=False)
ymin, ymax = ylim()
ylim(ymin, ymax*1.1)
tight_layout()
savefig("%s-%s.pdf" % (session, line))
if savedat:
np.savetxt('%s-%s.dat' % (session, line), np.vstack((cbins, gn)).T,
header='Energy (keV), Count', delimiter='\t')
def processor(args):
"""map-able function that processes a single frame of data
Argument: a single tuple composed of the following, in order
file_name : string, required
pointslist : list, required
file_number : ignored, optional
background : array, optional
donelist : list, optional
A list of files that have already been processed
status : ??????
"""
fname = args[0]
frame_num = int(path.splitext(path.split(fname)[-1])[0])
pointslist = args[1]
if len(args) > 2: i = args[2]
if len(args) > 3:
bgimg = args[3]
else:
bgimg = None
if len(args) > 4:
donelist = args[4]
donelist.append(i)
if len(donelist)%1000 == 1:
sys.stderr.write("Completed %d\n"%(len(donelist)-1))
sys.stderr.write("Found %d holes\n" % len(pointslist))
if len(donelist)%10000 == 0 and len(pointslist) > 0:
xl, yl, varxl, varyl, el, xr, yr, varxr, varyr, er, framenum =\
zip(*list(pointslist))
savemat('tempfile.mat',
{'xl':xl, 'yl':yl, 'varxl': varxl, 'xr':xr, 'yr':yr,
'el': el, 'er': er, 'varxr': varxr, 'framenum':framenum},
oned_as = 'row')
sys.stderr.write("Saved a snapshot\n")
if len(args) > 5:
THRESH = args[5]
# Finally, load in the image
try:
img1, img2 = Analysis.loadsplit(fname, bgimg = bgimg)
except IOError:
print "Failed loading!"
return
pointslist.extend(Analysis.findpairs(img1, img2, thresh=THRESH,
DEBUG=DEBUG, abs_thresh=True,
frame_num = frame_num))
def __init__(self, parent, controller):
tk.Frame.__init__(self, parent)
img = self.img = ImageTk.PhotoImage(Image.open('build_your_career.jpg'))
panel = Tkinter.Label(self,image=img)
panel.place(x="0",y="0")
scrollbar = tk.Scrollbar(self)
scrollbar.pack(side="right", fill="y")
result = Analysis.infoOLD()
text_field = "Quiz: Which Major Is Right For You?\nYour last result: "+result+"\nSource: http://getcollegecredit.com/blog/article/which_major_is_right_for_you\n"
f = open('quiz.txt','r')
text_field = text_field + str(f.read())
text = tk.Text(self)
text.insert("end",text_field)
text.pack()
text.config(state="disabled",yscrollcommand=scrollbar.set)
scrollbar.config(command=text.yview)
self.entryVariable = Tkinter.StringVar()
self.entry = Tkinter.Entry(self,textvariable=self.entryVariable, width = 50)
self.entry.bind("<Return>", self.OnPressEnter)
self.entryVariable.set(u"Ex: 1 D,2 C, ...")
self.entry.pack()
self.labelVariable = Tkinter.StringVar()
label = Tkinter.Label(self,textvariable=self.labelVariable,justify="left",wraplength=500,fg="yellow",bg="blue",width = 80,height=10)
self.labelVariable.set(u"Hello!")
label.pack()
button1 = tk.Button(self, text="Back to Home",
command=lambda: controller.show_frame(HomePage))
button1.pack()
def __init__(self, data, filename, view, parent):
super(DisassemblerView, self).__init__(parent)
self.status = ""
self.view = view
self.data = data
for type in ExeFormats:
exe = type(data)
if exe.valid:
self.data = exe
self.view.exe = exe
break
# Create analysis and start it in another thread
self.analysis = Analysis(self.data)
self.analysis_thread = threading.Thread(None, self.analysis_thread_proc)
self.analysis_thread.daemon = True
self.analysis_thread.start()
# Start disassembly view at the entry point of the binary
if hasattr(self.data, "entry"):
self.function = self.data.entry()
else:
self.function = None
self.update_id = None
self.ready = False
self.desired_pos = None
self.highlight_token = None
self.cur_instr = None
self.scroll_mode = False
self.blocks = {}
self.show_il = False
self.simulation = None
# Create timer to automatically refresh view when it needs to be updated
self.updateTimer = QTimer()
self.updateTimer.setInterval(100)
self.updateTimer.setSingleShot(False)
self.updateTimer.timeout.connect(self.updateTimerEvent)
self.updateTimer.start()
self.initFont()
# Initialize scroll bars
self.width = 0
self.height = 0
self.setHorizontalScrollBarPolicy(Qt.ScrollBarAsNeeded)
self.setVerticalScrollBarPolicy(Qt.ScrollBarAsNeeded)
self.horizontalScrollBar().setSingleStep(self.charWidth)
self.verticalScrollBar().setSingleStep(self.charHeight)
areaSize = self.viewport().size()
self.adjustSize(areaSize.width(), areaSize.height())
# Setup navigation
self.view.register_navigate("disassembler", self, self.navigate)
self.view.register_navigate("make_proc", self, self.make_proc)
self.search_regex = None
self.last_search_type = FindDialog.SEARCH_HEX
def analyzeParsedSmali(classes,sharedobjs):
# Begin Analysis
log.info("Analysis Started")
dependencies = {}
dependencies["all"],dependencies["internal"],dependencies["external"],dependencies["unknown"] = getDependencies(classes)
results = Analysis.runAnalysis(classes,dependencies,sharedobjs)
log.saveLibrariesToPickle(dependencies["all"])
return results
def get_total_SSRO_events(pqf, RO_start, RO_length, marker_chan, chan_rnd_0, chan_rnd_1, sync_time_lim, VERBOSE = True):
"""
Returns SSRO data for all marked events.
Colums are:
Sync Nymber | number of photons | RND number indicator | RND number | Sync Time RND number | Sync Times photon 1-24 |
"""
_a = get_attributes(pqf)
# Gets the number of blocks in the data
num_blocks = Analysis.get_num_blocks(pqf)
if VERBOSE or (num_blocks > 1):
print 'The total number of blocks is:', num_blocks
# Initializes arrays to save the PQ-data
PQ_sync_number = np.empty((0,), dtype = np.uint32)
PQ_special = np.empty((0,), dtype = np.uint32)
PQ_sync_time = np.empty((0,), dtype = np.uint64)
PQ_time = np.empty((0,), dtype = np.uint64)
PQ_channel = np.empty((0,), dtype = np.uint32)
# Initializes an array to save the SSRO data
total_SSRO_events = np.empty((0,30), dtype = np.uint64)
# Loops over every block
for i in range(num_blocks):
# Get the SSRO events and PQ data for these sync numbers
gc.collect()
_events, _PQ_sync_number, _PQ_special, _PQ_sync_time, _PQ_time, _PQ_channel = \
get_SSRO_events(pqf, marker_chan, RO_start, RO_length, chan_rnd_0, chan_rnd_1, sync_time_lim = sync_time_lim, index = i+1, VERBOSE = VERBOSE)
# Concatenates all PQ data
PQ_sync_number = np.hstack((PQ_sync_number,_PQ_sync_number))
PQ_special = np.hstack((PQ_special, _PQ_special))
PQ_sync_time = np.hstack((PQ_sync_time, _PQ_sync_time))
PQ_time = np.hstack((PQ_time, _PQ_time))
PQ_channel = np.hstack((PQ_channel, _PQ_channel))
# Stacks all SSRO data
total_SSRO_events = np.vstack((total_SSRO_events, _events))
if VERBOSE>2:
print
print 'Found {} valid marked SSRO events in block'.format(int(len(_events))), i+1
print '===================================='
print
if VERBOSE:
print
print 'Found {} valid marked SSRO events in all blocks'.format(int(len(total_SSRO_events)))
print '===================================='
print
return total_SSRO_events, _a, PQ_sync_number, PQ_special, PQ_sync_time, PQ_time, PQ_channel
def __init__(self, P, lstOfGenerationsNumbers):
outputPath = os.path.dirname(os.path.realpath(__file__)) + "/"
f = open(outputPath+"Stats.txt","w")
f.write(" ====================================================== \n")
f.write("|| ||\n")
f.write("|| Statistical Properties of this simulation run ||\n")
f.write("|| ||\n")
f.write(" ====================================================== \n\n\n")
f.write("Number of Agents : {:>5}\n".format(P.numberAgents))
f.write("Maximum size of social groups : {:>5}\n".format(P.maxSocSize))
f.write("Maximum number of Generations : {:>5}\n".format(P.generations))
f.write("Number of individual CultEvo runs: {:>5}\n\n\n\n".format(P.numberOfSimulationRuns))
f.write("Values:\n")
f.write("=======\n\n")
f.write("Averages and measures of central location\n")
f.write(".........................................\n\n")
f.write("mean : {:>6.2f}\n".format(stat.mean(lstOfGenerationsNumbers)))
f.write("median : {:>6.2f}\n".format(stat.median(lstOfGenerationsNumbers)))
f.write("mean_low : {:>6.2f}\n".format(stat.median_low(lstOfGenerationsNumbers)))
f.write("mean_high : {:>6.2f}\n".format(stat.median_high(lstOfGenerationsNumbers)))
f.write("mean_grouped: {:>6.2f}\n".format(stat.median_grouped(lstOfGenerationsNumbers)))
try:
f.write("mode : {:>6.2f}\n\n\n\n".format(stat.mode(lstOfGenerationsNumbers) ))
except :
f.write("mode : two equal values found\n")
f.write("Measures of spread\n")
f.write("..................\n\n")
f.write("Population standard deviation of data: {0:>6.2f}\n".format(stat.pstdev(lstOfGenerationsNumbers)))
f.write("Population variance of data : {0:>6.2f}\n".format(stat.pvariance(lstOfGenerationsNumbers)))
f.write("Sample standard deviation of data : {0:>6.2f}\n".format(stat.stdev(lstOfGenerationsNumbers)))
f.write("Sample variance of data : {0:>6.2f}\n".format(stat.variance(lstOfGenerationsNumbers)))
f.close()
def OnPressEnter(self,event):
solution = self.entryVariable.get()
data_point = rw_csv.readStudyHrs()
x = []
y = []
for point in data_point:
x.append(int(point[0]))
y.append(int(point[1]))
(anal,predict_value) = Analysis.regression(x,y,int(solution))
self.text.insert("end",anal+"You should study for "+str(predict_value)+" hours for next exam!!!")
self.text.pack()
self.entry.focus_set()
self.entry.selection_range(0, Tkinter.END)
def __init__(self):
def nothing(x):
pass
self.node_name = "cv_bridge_demo"
rospy.init_node(self.node_name)
self.BoW = BoW.BoW("/home/iglu/catkin_ws/src/RGBDHand/src/bof.pkl")
Analysis.initial()
# What we do during shutdown
rospy.on_shutdown(self.cleanup)
# Create the OpenCV display window for the RGB image
self.cv_window_name = self.node_name
cv.NamedWindow(self.cv_window_name, cv.CV_WINDOW_NORMAL)
cv.MoveWindow(self.cv_window_name, 25, 75)
# And one for the depth image
cv.NamedWindow("Depth Image", cv.CV_WINDOW_NORMAL)
cv.MoveWindow("Depth Image", 25, 350)
# And one for the depth image
cv.NamedWindow("Histogram", cv.CV_WINDOW_NORMAL)
cv.MoveWindow("Histogram", 480, 350)
self.bridge = CvBridge()
self.Im_p = Image_Process.Image_Process()
# Subscribe to the camera image and depth topics and set
# the appropriate callbacks
self.RGB_sub = message_filters.Subscriber('/camera/rgb/image_raw', Image)
self.Depth_sub = message_filters.Subscriber('/camera/depth/image_raw', Image)
# rospy.Subscriber('Scene',String,callbackS)
self.ts = message_filters.ApproximateTimeSynchronizer([self.RGB_sub, self.Depth_sub], 1,1)
# self.image_sub = rospy.Subscriber("/camera/rgb/image_rect_color", Image, self.image_callback)
# self.depth_sub = rospy.Subscriber("/camera/depth/image_raw", Image, self.depth_callback)
self.ts.registerCallback(self.image_callback)
self.depth = np.zeros((480,680))
rospy.loginfo("Waiting for image topics...")
def _line_fit(data, min, line):
# Fit
(dE, width), (dE_error, width_error), e = Analysis.fit(data, binsize=binsize, min=min, line=line, shift=shift, method=method)
if method == "cs":
chi_squared, dof = e
if method in ("mle", "ls"):
print "%s: %.2f +/- %.2f eV @ Ec%+.2f eV" \
% (line, width, width_error, dE)
elif method == "cs":
print "%s: %.2f +/- %.2f eV @ Ec%+.2f eV (Red. chi^2 = %.1f/%d = %.2f)" \
% (line, width, width_error, dE, chi_squared, dof, chi_squared/dof)
return dE, width, width_error
def __init__(self,B):
path = "/media/iglu/Data/Dataset/DatasetIglu/Dataset_cleaned_v4"
i = 0
Analysis.initial()
files = open("Output_1_L.txt", 'w')
start_time = timeit.default_timer()
for x in os.listdir(path):
# if i == 0:
# i += 1
# continue
# else:
f = open(path+"/"+x+"/List.txt",'r')
for line in f:
Time = line
file1 = next(f).rstrip('\n')
file2 = next(f).rstrip('\n')
Label = next(f).rstrip('\n')
RGB = cv2.imread(path+"/"+x+"/RGB/"+file1)
Depth = cv2.imread(path+"/"+x+"/Depth/"+file2)
# files.write(RGB.__str__())
# files.write(Depth.__str__())
# files.write(Label+" "+x+" "+file1+"\n")
Analysis.addDepth(Depth)
Analysis.addRGB(RGB)
Analysis.addScene(Label)
i += 1
print "Enviado "+x
# if i > 150:
# break
elapsed = timeit.default_timer() - start_time
print "Tiempo: "+elapsed.__str__()
print "Starting Training"
start_time = timeit.default_timer()
Analysis.CompleteAnalysis(B,True,0.3,files)
files.close()
#Analysis.Mostrar()
# code you want to evaluate
elapsed = timeit.default_timer() - start_time
print "Tiempo: "+elapsed.__str__()
def dumbClustering(wordList, fName = 'clusters.txt'):
"""
Clusters by starting each comment as it's own cluster, and seeing which other clusters
have a lot of words in common with it
No stemming, all words are included
It's pretty bad, really.
"""
#make clusters
allClusters = []
for c in wordList:
tempCluster = Cluster()
freqTuple = Analysis.wordFrequency([c])
if len(freqTuple) == 0:
continue
freqDict = {}
for w, f in freqTuple:
freqDict[w] = f
tempCluster.addComment(c, freqDict)
allClusters.append(tempCluster)
#play with these values
start = 1.0
mid = .6
end = .3
step = -1.0/15
#combine clusters
dumbClusterStep(start, mid, step, allClusters)
singleClusters = Cluster()
for j in range(len(allClusters)-1, -1, -1):
if allClusters[j].numComments() <= 5:
singleClusters.addCluster(allClusters[j])
del allClusters[j]
allClusters.append(singleClusters)
dumbClusterStep(mid, end, step, allClusters)
f = open(fName, 'w')
for c in allClusters:
f.write(str(c)+'\n')
f.close()
def basic(constants): """ Runs a basic experiment Parameters: -``constants``: Config settings for the EA from the config files specified at run time """ try: random.seed(constants["seed"]) except KeyError: pass results = [] for run in range(constants["runs"]): constants["runNumber"] = run evaluation = Util.moduleClasses(Fitness)[constants["problem"]](constants).eval results.append(oneRun(constants, evaluation)) data = [] for i in results: data.append(i[-1][1]) return Analysis.meanstd(data), results
def get_total_SSRO_events_quick(pqf, RO_start, RO_length, marker_chan, sync_time_lim, VERBOSE = True):
"""
Returns quick SSRO data for all marked events.
Sync Nymber | number of photons | Sync time first photon
"""
columns = "Sync_Number,Number of photons,Sync_Time_photon_1"
_a = {'Columns': columns}
# Gets the number of blocks for the data
num_blocks = Analysis.get_num_blocks(pqf)
if VERBOSE:
print 'The total number of blocks is:', num_blocks
# Initializes the array to save all SSRO events
total_SSRO_events = np.empty((0,3))
# Loop over all blocks
for i in range(num_blocks):
_events = get_SSRO_events_quick(pqf, marker_chan, RO_start, RO_length, sync_time_lim = sync_time_lim, index = i+1)
# Stacks all events for several blocks
total_SSRO_events = np.vstack((total_SSRO_events, _events))
if VERBOSE:
print
print 'Found {} valid marked SSRO events in block'.format(int(len(_events))), i+1
print '===================================='
print
if VERBOSE:
print
print 'Found {} valid marked SSRO events in all blocks'.format(int(len(total_SSRO_events)))
print '===================================='
print
return total_SSRO_events, _a
def handleCmdButton4(self):
#check to see if data has been loaded
test = self.checkData()
if( test == False):
return
self.size = None
self.color = None
#check to see if there is any data in the set after filters applied
if( self.dataObject.filtersize() == 0):
tkMessageBox.showwarning("No data!","Remove Filters from the data or load a different data set!")
return
print "handling command button 4"
# run pca on all of the numerical data
self.Analysis = Analysis(name = "PCA", data = self.dataObject.getData_num())
#dictioanry that will store the headers for the pca analsysis plots
self.pcadataHeader = {}
# this will run the dialog for pca
inputBox = PCADialog(parent = self.root, pcaDataHeader = self.pcadataHeader, displayClass = self, Analysis = self.Analysis)
if( self.pcadataHeader['check'] == True):
self.xAxisLabel.set( self.pcadataHeader.get('x') )
self.yAxisLabel.set( self.pcadataHeader.get('y') )
z = self.pcadataHeader.get('z')
#z is optional
if( z == None ):
z = ""
self.zAxisLabel.set(z)
self.buildLabels()
#graphs the pca points
self.buildPCA()
if( self.pcadataHeader.get('saveData') ):
filename = self.pcadataHeader.get('filename')
filename = filename + ".csv"
self.Analysis.writeData(filename = filename)
limit_inner = None
if len(sys.argv) == 5:
if sys.argv[4] != 'None':
limit_inner = float(sys.argv[4])
fname = basedir + '/' + tag + '_%i_XCO_P8_limit_inner.hdf5' % limit_inner
fix_xco = False
if len(sys.argv) == 6:
if sys.argv[5] == '0':
fix_xco == True
# Load the analysis
A = Analysis.Analysis(
tag='P7REP_CLEAN_V15_calore',
fglpath='/pfs/carlson/gll_psc_v14.fit',
templateDir='/home/carlson/pfs/Extended_archive_v15/Templates',
basepath='/pfs/carlson/GCE_sys/')
# A.GenPointSourceTemplate(pscmap=(A.basepath + '/PSC_all_sky_3fgl.npy'))
# A.BinPhotons(outfile='binned_photons_all_sky.npy')
#A.GenSquareMask(l_range=[-180.,180], b_range=[-40.,40.], plane_mask=1.)
A.BinPhotons(infile='binned_photons_all_sky.npy')
# Load 2FGL
A.AddPointSourceTemplate(fixNorm=True, pscmap=('PSC_3FGL_with_ext.npy'))
A.CalculatePixelWeights(diffuse_model='fermi_diffuse_' + A.tag + '.npy',
psc_model='PSC_3FGL_with_ext.npy',
alpha_psc=5.,
f_psc=0.2)
A.AddIsotropicTemplate(
fixNorm=False, fixSpectrum=True
) # External chi^2 used to fix normalization within uncertainties
Drawing.DrawLetter(Folder, Filename)
# if there are three arguments set the file and folder name
if len(sys.argv) == 3:
if sys.argv[1] == "alphabet":
DrawingsPerLetter = int(sys.argv[2])
for Item in listdir("./Letters/"):
for i in range(0, DrawingsPerLetter):
Folder = Item
SaveAsUnusedName()
elif sys.argv[1] == "analyze":
# actually do the drawing
Folder = sys.argv[2]
SaveAsUnusedName()
# get the analytics data
AnalysisData = Analysis.RankingsFromFile(Folder, Filename, "FullCharacterRecognition")
for i in range(0, 3):
print AnalysisData[i]["Value"], "letter", AnalysisData[i]["Letter"]
if AnalysisData[0]["Letter"] == Folder:
os.remove("Letters/" + Folder + "/" + Filename)
print "removing reduntant sample."
else:
Filename = sys.argv[2]
Folder = sys.argv[1]
# actually do the drawing
Drawing.DrawLetter(Folder, Filename)
import os
import Analysis
Count = 0
Path = "E:/Data/PageSources"
Data = os.listdir(Path)
for i in Data:
InData = os.listdir(Path + "/" + i)
for j in InData:
Count += 1
print(Count)
InPath = Path + "/" + i + "/" + j
Analysis.FunctionGamda(InPath, i)
entree_co_labels = pickEntree_df.axes[0].tolist()
entree_company = entree_co_labels[0]
matchFoundUpdate(entree_company, preference)
matchFound = 1
if matchFound == 0: # all solutions have failed
entree_dict['Entree Seating:'].append("failed")
dessert_dict['Dessert Seating:'].append("failed")
# Now, Combine all of the dictionaries created into a data frame
results = Merge(firstName_dict, lastName_dict, eID_dict, year_dict,
email_dict, entree_dict, dessert_dict, major_dict)
results_df = pd.DataFrame(results)
# Create a data frame of only the successful assignments
assignments_df = results_df[~(
(results_df['Entree Seating:'].str.contains('failed')))]
# Create a Pandas Excel writer using XlsxWriter as the engine.
writer = pd.ExcelWriter('Results_2018.xlsx', engine='xlsxwriter')
# Convert the assignments dataframe to an XlsxWriter Excel object.
results_df.to_excel(writer, sheet_name='All Results')
assignments_df.to_excel(writer, sheet_name='Successful Assignments')
# Close the Pandas Excel writer and output the Excel file.
writer.save()
import Analysis
Analysis.analyze_preferences(preference_dessert_dict,
preference_entree_dict)
Analysis.analyze_major_dist(assignments_df, results_df)
import Analysis
if __name__ == '__main__':
totalReceiver = int(input('Enter number of receivers : '))
wTable = WalshTableGenerator.getWalshTable(totalReceiver)
output_file = "to.txt"
receiverThreadPool = []
receiverList = []
for index in range(0, totalReceiver):
new_receiver = Receiver.Receiver(index, output_file, wTable[index])
receiverList.append(new_receiver)
new_receiver_thread = threading.Thread(
target=new_receiver.startReceive, args=())
receiverThreadPool.append(new_receiver_thread)
new_receiver_thread.start()
for index in range(0, totalReceiver):
receiverThreadPool[index].join()
pktCount = 0
totalTime = 0
for index in range(0, totalReceiver):
pktCount += receiverList[index].report.pktCount
totalTime += receiverList[index].report.totalTime
totalTime /= totalReceiver
finalReport = Analysis.Report(pktCount, totalTime)
finalReport.storeReport(totalReceiver)
class DisassemblerView(QAbstractScrollArea):
statusUpdated = Signal(QWidget, name="statusUpdated")
def __init__(self, data, filename, view, parent):
super(DisassemblerView, self).__init__(parent)
self.status = ""
self.view = view
self.data = data
for type in ExeFormats:
exe = type(data)
if exe.valid:
self.data = exe
self.view.exe = exe
break
# Create analysis and start it in another thread
self.analysis = Analysis(self.data)
self.analysis_thread = threading.Thread(None,
self.analysis_thread_proc)
self.analysis_thread.daemon = True
self.analysis_thread.start()
# Start disassembly view at the entry point of the binary
if hasattr(self.data, "entry"):
self.function = self.data.entry()
else:
self.function = None
self.update_id = None
self.ready = False
self.desired_pos = None
self.highlight_token = None
self.cur_instr = None
self.scroll_mode = False
self.blocks = {}
self.show_il = False
self.simulation = None
# Create timer to automatically refresh view when it needs to be updated
self.updateTimer = QTimer()
self.updateTimer.setInterval(100)
self.updateTimer.setSingleShot(False)
self.updateTimer.timeout.connect(self.updateTimerEvent)
self.updateTimer.start()
self.initFont()
# Initialize scroll bars
self.width = 0
self.height = 0
self.setHorizontalScrollBarPolicy(Qt.ScrollBarAsNeeded)
self.setVerticalScrollBarPolicy(Qt.ScrollBarAsNeeded)
self.horizontalScrollBar().setSingleStep(self.charWidth)
self.verticalScrollBar().setSingleStep(self.charHeight)
areaSize = self.viewport().size()
self.adjustSize(areaSize.width(), areaSize.height())
# Setup navigation
self.view.register_navigate("disassembler", self, self.navigate)
self.view.register_navigate("make_proc", self, self.make_proc)
self.search_regex = None
self.last_search_type = FindDialog.SEARCH_HEX
def initFont(self):
# Get font and compute character sizes
self.font = getMonospaceFont()
self.baseline = int(QFontMetricsF(self.font).ascent())
self.charWidth = QFontMetricsF(self.font).width('X')
self.charHeight = int(QFontMetricsF(
self.font).height()) + getExtraFontSpacing()
self.charOffset = getFontVerticalOffset()
def adjustSize(self, width, height):
# Recompute size information
self.renderWidth = self.width
self.renderHeight = self.height
self.renderXOfs = 0
self.renderYOfs = 0
if self.renderWidth < width:
self.renderXOfs = int((width - self.renderWidth) / 2)
self.renderWidth = width
if self.renderHeight < height:
self.renderYOfs = int((height - self.renderHeight) / 2)
self.renderHeight = height
# Update scroll bar information
self.horizontalScrollBar().setPageStep(width)
self.horizontalScrollBar().setRange(0, self.renderWidth - width)
self.verticalScrollBar().setPageStep(height)
self.verticalScrollBar().setRange(0, self.renderHeight - height)
def resizeEvent(self, event):
# Window was resized, adjust scroll bar
self.adjustSize(event.size().width(), event.size().height())
def get_cursor_pos(self):
if self.cur_instr is None:
return self.function
return self.cur_instr
def set_cursor_pos(self, addr):
if not self.view.navigate("disassembler", addr):
self.view_in_hex_editor(addr)
def get_selection_range(self):
return (self.get_cursor_pos(), self.get_cursor_pos())
def set_selection_range(self, begin, end):
self.set_cursor_pos(begin)
def write(self, data):
pos = self.get_cursor_pos()
if pos is None:
return False
return self.data.write(pos, data) == len(data)
def copy_address(self):
clipboard = QApplication.clipboard()
clipboard.clear()
mime = QMimeData()
mime.setText("0x%x" % self.get_cursor_pos())
clipboard.setMimeData(mime)
def analysis_thread_proc(self):
self.analysis.analyze()
def closeRequest(self):
# Stop analysis when closing tab
self.analysis.stop()
return True
def paintEvent(self, event):
# Initialize painter
p = QPainter(self.viewport())
p.setFont(self.font)
xofs = self.horizontalScrollBar().value()
yofs = self.verticalScrollBar().value()
if not self.ready:
# Analysis for the current function is not yet complete, paint loading screen
gradient = QLinearGradient(
QPointF(0, 0),
QPointF(self.viewport().size().width(),
self.viewport().size().height()))
gradient.setColorAt(0, QColor(232, 232, 232))
gradient.setColorAt(1, QColor(192, 192, 192))
p.setPen(QColor(0, 0, 0, 0))
p.setBrush(QBrush(gradient))
p.drawRect(0, 0,
self.viewport().size().width(),
self.viewport().size().height())
if self.function is None:
text = "No function selected"
else:
text = "Loading..."
p.setPen(Qt.black)
p.drawText((self.viewport().size().width() / 2) -
((len(text) * self.charWidth) / 2),
(self.viewport().size().height() / 2) +
self.charOffset + self.baseline - (self.charHeight / 2),
text)
return
# Render background
gradient = QLinearGradient(
QPointF(-xofs, -yofs),
QPointF(self.renderWidth - xofs, self.renderHeight - yofs))
gradient.setColorAt(0, QColor(232, 232, 232))
gradient.setColorAt(1, QColor(192, 192, 192))
p.setPen(QColor(0, 0, 0, 0))
p.setBrush(QBrush(gradient))
p.drawRect(0, 0,
self.viewport().size().width(),
self.viewport().size().height())
p.translate(self.renderXOfs - xofs, self.renderYOfs - yofs)
# Render each node
for block in self.blocks.values():
# Render shadow
p.setPen(QColor(0, 0, 0, 0))
p.setBrush(QColor(0, 0, 0, 128))
p.drawRect(block.x + self.charWidth + 4,
block.y + self.charWidth + 4,
block.width - (4 + 2 * self.charWidth),
block.height - (4 + 2 * self.charWidth))
# Render node background
gradient = QLinearGradient(
QPointF(0, block.y + self.charWidth),
QPointF(0, block.y + block.height - self.charWidth))
gradient.setColorAt(0, QColor(255, 255, 252))
gradient.setColorAt(1, QColor(255, 255, 232))
p.setPen(Qt.black)
p.setBrush(QBrush(gradient))
p.drawRect(block.x + self.charWidth, block.y + self.charWidth,
block.width - (4 + 2 * self.charWidth),
block.height - (4 + 2 * self.charWidth))
if self.cur_instr != None:
y = block.y + (2 * self.charWidth) + (
len(block.block.header_text.lines) * self.charHeight)
for instr in block.block.instrs:
if instr.addr == self.cur_instr:
p.setPen(QColor(0, 0, 0, 0))
p.setBrush(QColor(255, 255, 128, 128))
p.drawRect(block.x + self.charWidth + 3, y,
block.width - (10 + 2 * self.charWidth),
len(instr.text.lines) * self.charHeight)
y += len(instr.text.lines) * self.charHeight
if self.highlight_token:
# Render highlighted tokens
x = block.x + (2 * self.charWidth)
y = block.y + (2 * self.charWidth)
for line in block.block.header_text.tokens:
for token in line:
if token[2:] == self.highlight_token:
p.setPen(QColor(0, 0, 0, 0))
p.setBrush(QColor(192, 0, 0, 64))
p.drawRect(x + token[0] * self.charWidth, y,
token[1] * self.charWidth,
self.charHeight)
y += self.charHeight
for instr in block.block.instrs:
for line in instr.text.tokens:
for token in line:
if token[2:] == self.highlight_token:
p.setPen(QColor(0, 0, 0, 0))
p.setBrush(QColor(192, 0, 0, 64))
p.drawRect(x + token[0] * self.charWidth, y,
token[1] * self.charWidth,
self.charHeight)
y += self.charHeight
# Render node text
x = block.x + (2 * self.charWidth)
y = block.y + (2 * self.charWidth)
for line in block.block.header_text.lines:
partx = x
for part in line:
p.setPen(part[1])
p.drawText(partx, y + self.charOffset + self.baseline,
part[0])
partx += len(part[0]) * self.charWidth
y += self.charHeight
for instr in block.block.instrs:
for line in instr.text.lines:
partx = x
for part in line:
p.setPen(part[1])
p.drawText(partx, y + self.charOffset + self.baseline,
part[0])
partx += len(part[0]) * self.charWidth
y += self.charHeight
# Render edges
for edge in block.edges:
p.setPen(edge.color)
p.setBrush(edge.color)
p.drawPolyline(edge.polyline)
p.drawConvexPolygon(edge.arrow)
def isMouseEventInBlock(self, event):
# Convert coordinates to system used in blocks
xofs = self.horizontalScrollBar().value()
yofs = self.verticalScrollBar().value()
x = event.x() + xofs - self.renderXOfs
y = event.y() + yofs - self.renderYOfs
# Check each block for hits
for block in self.blocks.values():
# Compute coordinate relative to text area in block
blockx = x - (block.x + (2 * self.charWidth))
blocky = y - (block.y + (2 * self.charWidth))
# Check to see if click is within bounds of block
if (blockx < 0) or (blockx > (block.width - 4 * self.charWidth)):
continue
if (blocky < 0) or (blocky > (block.height - 4 * self.charWidth)):
continue
return True
return False
def getInstrForMouseEvent(self, event):
# Convert coordinates to system used in blocks
xofs = self.horizontalScrollBar().value()
yofs = self.verticalScrollBar().value()
x = event.x() + xofs - self.renderXOfs
y = event.y() + yofs - self.renderYOfs
# Check each block for hits
for block in self.blocks.values():
# Compute coordinate relative to text area in block
blockx = x - (block.x + (2 * self.charWidth))
blocky = y - (block.y + (2 * self.charWidth))
# Check to see if click is within bounds of block
if (blockx < 0) or (blockx > (block.width - 4 * self.charWidth)):
continue
if (blocky < 0) or (blocky > (block.height - 4 * self.charWidth)):
continue
# Compute row within text
row = int(blocky / self.charHeight)
# Determine instruction for this row
cur_row = len(block.block.header_text.lines)
if row < cur_row:
return block.block.entry
for instr in block.block.instrs:
if row < cur_row + len(instr.text.lines):
return instr.addr
cur_row += len(instr.text.lines)
return None
def getTokenForMouseEvent(self, event):
# Convert coordinates to system used in blocks
xofs = self.horizontalScrollBar().value()
yofs = self.verticalScrollBar().value()
x = event.x() + xofs - self.renderXOfs
y = event.y() + yofs - self.renderYOfs
# Check each block for hits
for block in self.blocks.values():
# Compute coordinate relative to text area in block
blockx = x - (block.x + (2 * self.charWidth))
blocky = y - (block.y + (2 * self.charWidth))
# Check to see if click is within bounds of block
if (blockx < 0) or (blockx > (block.width - 4 * self.charWidth)):
continue
if (blocky < 0) or (blocky > (block.height - 4 * self.charWidth)):
continue
# Compute row and column within text
col = int(blockx / self.charWidth)
row = int(blocky / self.charHeight)
# Check tokens to see if one was clicked
cur_row = 0
for line in block.block.header_text.tokens:
if cur_row == row:
for token in line:
if (col >= token[0]) and (col < (token[0] + token[1])):
# Clicked on a token
return token
cur_row += 1
for instr in block.block.instrs:
for line in instr.text.tokens:
if cur_row == row:
for token in line:
if (col >= token[0]) and (col <
(token[0] + token[1])):
# Clicked on a token
return token
cur_row += 1
return None
def find_instr(self, addr):
for block in self.blocks.values():
for instr in block.block.instrs:
if instr.addr == addr:
return instr
return None
def nop_out(self, addr):
instr = self.find_instr(addr)
if instr != None:
self.view.begin_undo()
instr.patch_to_nop(self.data)
self.view.commit_undo()
def always_branch(self, addr):
instr = self.find_instr(addr)
if instr != None:
self.view.begin_undo()
instr.patch_to_always_branch(self.data)
self.view.commit_undo()
def invert_branch(self, addr):
instr = self.find_instr(addr)
if instr != None:
self.view.begin_undo()
instr.patch_to_invert_branch(self.data)
self.view.commit_undo()
def skip_and_return_zero(self, addr):
instr = self.find_instr(addr)
if instr != None:
self.view.begin_undo()
instr.patch_to_zero_return(self.data)
self.view.commit_undo()
def skip_and_return_value(self, addr):
instr = self.find_instr(addr)
if instr != None:
value, ok = QInputDialog.getText(self, "Skip and Return Value",
"Return value:", QLineEdit.Normal)
if ok:
try:
value = int(value, 0)
except:
QMessageBox.critical(self, "Error",
"Expected numerical address")
return
self.view.begin_undo()
instr.patch_to_fixed_return_value(self.data, value)
self.view.commit_undo()
def view_in_hex_editor(self, addr):
if not self.view.navigate("exe", addr):
self.view.navigate("hex", addr)
def show_address(self):
if "address" in self.analysis.options:
addr = False
else:
addr = True
self.analysis.set_address_view(addr)
def context_menu(self, addr):
popup = QMenu()
view_in_hex = popup.addAction("View in &hex editor")
view_in_hex.triggered.connect(lambda: self.view_in_hex_editor(addr))
view_in_hex.setShortcut(QKeySequence(Qt.Key_H))
popup.addAction("Copy address", self.copy_address)
enter_name_action = popup.addAction("Re&name symbol", self.enter_name)
enter_name_action.setShortcut(QKeySequence(Qt.Key_N))
undefine_name_action = popup.addAction("&Undefine symbol",
self.undefine_name)
undefine_name_action.setShortcut(QKeySequence(Qt.Key_U))
show_address_action = popup.addAction("Show &address",
self.show_address)
show_address_action.setCheckable(True)
show_address_action.setChecked("address" in self.analysis.options)
popup.addSeparator()
patch = popup.addMenu("&Patch")
patch.addAction("Convert to NOP").triggered.connect(
lambda: self.nop_out(addr))
instr = self.find_instr(addr)
if instr:
if instr.is_patch_branch_allowed():
patch.addAction("Never branch").triggered.connect(
lambda: self.nop_out(addr))
patch.addAction("Always branch").triggered.connect(
lambda: self.always_branch(addr))
patch.addAction("Invert branch").triggered.connect(
lambda: self.invert_branch(addr))
if instr.is_patch_to_zero_return_allowed():
patch.addAction("Skip and return zero").triggered.connect(
lambda: self.skip_and_return_zero(addr))
if instr.is_patch_to_fixed_return_value_allowed():
patch.addAction("Skip and return value...").triggered.connect(
lambda: self.skip_and_return_value(addr))
popup.exec_(QCursor.pos())
def mousePressEvent(self, event):
if (event.button() != Qt.LeftButton) and (event.button() !=
Qt.RightButton):
return
if not self.isMouseEventInBlock(event):
# Click outside any block, enter scrolling mode
self.scroll_base_x = event.x()
self.scroll_base_y = event.y()
self.scroll_mode = True
self.viewport().grabMouse()
return
# Check for click on a token and highlight it
token = self.getTokenForMouseEvent(event)
if token:
self.highlight_token = token[2:]
else:
self.highlight_token = None
# Update current instruction
instr = self.getInstrForMouseEvent(event)
if instr != None:
self.cur_instr = instr
else:
self.cur_instr = None
self.viewport().update()
if (instr != None) and (event.button() == Qt.RightButton):
self.context_menu(instr)
def mouseMoveEvent(self, event):
if self.scroll_mode:
x_delta = self.scroll_base_x - event.x()
y_delta = self.scroll_base_y - event.y()
self.scroll_base_x = event.x()
self.scroll_base_y = event.y()
self.horizontalScrollBar().setValue(
self.horizontalScrollBar().value() + x_delta)
self.verticalScrollBar().setValue(
self.verticalScrollBar().value() + y_delta)
def mouseReleaseEvent(self, event):
if event.button() != Qt.LeftButton:
return
if self.scroll_mode:
self.scroll_mode = False
self.viewport().releaseMouse()
def mouseDoubleClickEvent(self, event):
token = self.getTokenForMouseEvent(event)
if token and (token[2] == "ptr"):
self.analysis.lock.acquire()
if not self.analysis.functions.has_key(token[3]):
# Not a function or not analyzed, go to address in hex editor
addr = token[3]
self.analysis.lock.release()
self.view_in_hex_editor(addr)
else:
self.view.add_history_entry()
self.function = token[3]
self.ready = False
self.desired_pos = None
self.cur_instr = None
self.highlight_token = None
self.viewport().update()
self.analysis.lock.release()
def go_to_address(self):
addr_str, ok = QInputDialog.getText(self, "Go To Address", "Address:",
QLineEdit.Normal)
if ok:
try:
addr = int(addr_str, 16)
if (addr < self.data.start()) or (addr > self.data.end()):
if hasattr(self.data, "symbols_by_name") and (
addr_str in self.data.symbols_by_name):
addr = self.data.symbols_by_name[addr_str]
else:
QMessageBox.critical(self, "Error",
"Address out of range")
return
except:
if hasattr(self.data, "symbols_by_name") and (
addr_str in self.data.symbols_by_name):
addr = self.data.symbols_by_name[addr_str]
elif (addr_str[0] == '@') and hasattr(
self.data,
"symbols_by_name") and (addr_str[1:]
in self.data.symbols_by_name):
addr = self.data.symbols_by_name[addr_str[1:]]
else:
QMessageBox.critical(self, "Error",
"Invalid address or symbol")
return
# Try navigating within disassembly, if it isn't within a function then
# navigate to the hex editor
if not self.view.navigate("disassembler", addr):
self.view_in_hex_editor(addr)
def enter_name(self):
# A symbol must be selected
if (self.highlight_token
== None) or (self.highlight_token[0] != "ptr"):
QMessageBox.critical(self, "Error", "No symbol selected.")
return
addr = self.highlight_token[1]
name = self.highlight_token[2]
# Ask for new name
new_name, ok = QInputDialog.getText(self, "Rename Symbol",
"Symbol name:", QLineEdit.Normal,
name)
if ok:
self.analysis.create_symbol(addr, new_name)
def undefine_name(self):
# A symbol must be selected
if (self.highlight_token
== None) or (self.highlight_token[0] != "ptr"):
QMessageBox.critical(self, "Error", "No symbol selected.")
return
addr = self.highlight_token[1]
name = self.highlight_token[2]
# Ask for new name
self.analysis.undefine_symbol(addr, name)
def navigate_for_find(self, addr):
func, instr = self.analysis.find_instr(addr, True)
if func != None:
self.navigate(addr)
else:
self.make_proc(addr)
self.cur_instr = addr
self.desired_pos = None
def perform_find(self, dlg):
self.search_regex = dlg.search_regex()
if self.cur_instr != None:
self.search_start = self.cur_instr
else:
if self.function is None:
return
self.search_start = self.function
found_loc = self.data.find(self.search_regex, self.search_start)
if found_loc != -1:
self.view.add_history_entry()
self.navigate_for_find(found_loc)
self.search_pos = found_loc + 1
return
found_loc = self.data.find(self.search_regex, self.data.start())
if (found_loc != -1) and (found_loc < self.search_start):
self.view.add_history_entry()
self.navigate_for_find(found_loc)
self.search_pos = found_loc + 1
return
QMessageBox.information(self, "Not Found", "Search string not found.")
def find(self):
dlg = FindDialog(self.last_search_type, self)
if dlg.exec_() == QDialog.Accepted:
self.last_search_type = dlg.search_type()
self.perform_find(dlg)
def find_next(self):
if self.search_regex == None:
QMessageBox.critical(self, "Error", "No active search")
return
found_loc = self.data.find(self.search_regex, self.search_pos)
if self.search_pos >= self.search_start:
if found_loc != -1:
self.view.add_history_entry()
self.navigate_for_find(found_loc)
self.search_pos = found_loc + 1
return
self.search_pos = 0
else:
if (found_loc != -1) and (found_loc < self.search_start):
self.view.add_history_entry()
self.navigate_for_find(found_loc)
self.search_pos = found_loc + 1
return
QMessageBox.information(self, "End of Search",
"No additional matches found.")
self.search_pos = self.search_start
return
found_loc = self.data.find(self.search_regex, self.search_pos)
if found_loc < self.search_start:
self.view.add_history_entry()
self.navigate_for_find(found_loc)
self.search_pos = found_loc + 1
return
QMessageBox.information(self, "End of Search",
"No additional matches found.")
self.search_pos = self.search_start
def keyPressEvent(self, event):
if event.key() == Qt.Key_H:
if self.cur_instr != None:
self.view_in_hex_editor(self.cur_instr)
else:
if self.function is not None:
self.view_in_hex_editor(self.function)
elif event.key() == Qt.Key_G:
self.go_to_address()
elif event.key() == Qt.Key_N:
self.enter_name()
elif event.key() == Qt.Key_U:
self.undefine_name()
elif event.key() == Qt.Key_Slash:
dlg = FindDialog(FindDialog.SEARCH_REGEX, self)
if dlg.exec_() == QDialog.Accepted:
self.perform_find(dlg)
else:
super(DisassemblerView, self).keyPressEvent(event)
def prepareGraphNode(self, block):
# Compute size of node in pixels
width = 0
height = 0
for line in block.block.header_text.lines:
chars = 0
for part in line:
chars += len(part[0])
if chars > width:
width = chars
height += 1
for instr in block.block.instrs:
for line in instr.text.lines:
chars = 0
for part in line:
chars += len(part[0])
if chars > width:
width = chars
height += 1
block.width = (width + 4) * self.charWidth + 4
block.height = (height * self.charHeight) + (4 * self.charWidth) + 4
def adjustGraphLayout(self, block, col, row):
block.col += col
block.row += row
for edge in block.new_exits:
self.adjustGraphLayout(self.blocks[edge], col, row)
def computeGraphLayout(self, block):
# Compute child node layouts and arrange them horizontally
col = 0
row_count = 1
for edge in block.new_exits:
self.computeGraphLayout(self.blocks[edge])
self.adjustGraphLayout(self.blocks[edge], col, 1)
col += self.blocks[edge].col_count
if (self.blocks[edge].row_count + 1) > row_count:
row_count = self.blocks[edge].row_count + 1
block.row = 0
if col >= 2:
# Place this node centered over the child nodes
block.col = (col - 2) / 2
block.col_count = col
else:
# No child nodes, set single node's width (nodes are 2 columns wide to allow
# centering over a branch)
block.col = 0
block.col_count = 2
block.row_count = row_count
def isEdgeMarked(self, edges, row, col, index):
if index >= len(edges[row][col]):
return False
return edges[row][col][index]
def markEdge(self, edges, row, col, index):
while len(edges[row][col]) <= index:
edges[row][col] += [False]
edges[row][col][index] = True
def findHorizEdgeIndex(self, edges, row, min_col, max_col):
# Find a valid index
i = 0
while True:
valid = True
for col in range(min_col, max_col + 1):
if self.isEdgeMarked(edges, row, col, i):
valid = False
break
if valid:
break
i += 1
# Mark chosen index as used
for col in range(min_col, max_col + 1):
self.markEdge(edges, row, col, i)
return i
def findVertEdgeIndex(self, edges, col, min_row, max_row):
# Find a valid index
i = 0
while True:
valid = True
for row in range(min_row, max_row + 1):
if self.isEdgeMarked(edges, row, col, i):
valid = False
break
if valid:
break
i += 1
# Mark chosen index as used
for row in range(min_row, max_row + 1):
self.markEdge(edges, row, col, i)
return i
def routeEdge(self, horiz_edges, vert_edges, edge_valid, start, end,
color):
edge = DisassemblerEdge(color, end)
# Find edge index for initial outgoing line
i = 0
while True:
if not self.isEdgeMarked(vert_edges, start.row + 1, start.col + 1,
i):
break
i += 1
self.markEdge(vert_edges, start.row + 1, start.col + 1, i)
edge.addPoint(start.row + 1, start.col + 1)
edge.start_index = i
horiz = False
# Find valid column for moving vertically to the target node
if end.row < (start.row + 1):
min_row = end.row
max_row = start.row + 1
else:
min_row = start.row + 1
max_row = end.row
col = start.col + 1
if min_row != max_row:
ofs = 0
while True:
col = start.col + 1 - ofs
if col >= 0:
valid = True
for row in range(min_row, max_row + 1):
if not edge_valid[row][col]:
valid = False
break
if valid:
break
col = start.col + 1 + ofs
if col < len(edge_valid[min_row]):
valid = True
for row in range(min_row, max_row + 1):
if not edge_valid[row][col]:
valid = False
break
if valid:
break
ofs += 1
if col != (start.col + 1):
# Not in same column, need to generate a line for moving to the correct column
if col < (start.col + 1):
min_col = col
max_col = start.col + 1
else:
min_col = start.col + 1
max_col = col
index = self.findHorizEdgeIndex(horiz_edges, start.row + 1,
min_col, max_col)
edge.addPoint(start.row + 1, col, index)
horiz = True
if end.row != (start.row + 1):
# Not in same row, need to generate a line for moving to the correct row
index = self.findVertEdgeIndex(vert_edges, col, min_row, max_row)
edge.addPoint(end.row, col, index)
horiz = False
if col != (end.col + 1):
# Not in ending column, need to generate a line for moving to the correct column
if col < (end.col + 1):
min_col = col
max_col = end.col + 1
else:
min_col = end.col + 1
max_col = col
index = self.findHorizEdgeIndex(horiz_edges, end.row, min_col,
max_col)
edge.addPoint(end.row, end.col + 1, index)
horiz = True
# If last line was horizontal, choose the ending edge index for the incoming edge
if horiz:
index = self.findVertEdgeIndex(vert_edges, end.col + 1, end.row,
end.row)
edge.points[len(edge.points) - 1][2] = index
return edge
def renderFunction(self, func):
# Create render nodes
self.blocks = {}
for block in func.blocks.values():
self.blocks[block.entry] = DisassemblerBlock(block)
self.prepareGraphNode(self.blocks[block.entry])
# Populate incoming lists
for block in self.blocks.values():
for edge in block.block.exits:
self.blocks[edge].incoming += [block.block.entry]
# Construct acyclic graph where each node is used as an edge exactly once
block = func.blocks[func.entry]
visited = [func.entry]
queue = [self.blocks[func.entry]]
changed = True
while changed:
changed = False
# First pick nodes that have single entry points
while len(queue) > 0:
block = queue.pop()
for edge in block.block.exits:
if edge in visited:
continue
# If node has no more unseen incoming edges, add it to the graph layout now
if len(self.blocks[edge].incoming) == 1:
self.blocks[edge].incoming.remove(block.block.entry)
block.new_exits += [edge]
queue += [self.blocks[edge]]
visited += [edge]
changed = True
# No more nodes satisfy constraints, pick a node to continue constructing the graph
best = None
for block in self.blocks.values():
if not block.block.entry in visited:
continue
for edge in block.block.exits:
if edge in visited:
continue
if (best == None) or (len(
self.blocks[edge].incoming) < best_edges) or (
(len(self.blocks[edge].incoming) == best_edges)
and (edge < best)):
best = edge
best_edges = len(self.blocks[edge].incoming)
best_parent = block
if best != None:
self.blocks[best].incoming.remove(best_parent.block.entry)
best_parent.new_exits += [best]
visited += [best]
changed = True
# Compute graph layout from bottom up
self.computeGraphLayout(self.blocks[func.entry])
# Prepare edge routing
horiz_edges = [None] * (self.blocks[func.entry].row_count + 1)
vert_edges = [None] * (self.blocks[func.entry].row_count + 1)
edge_valid = [None] * (self.blocks[func.entry].row_count + 1)
for row in range(0, self.blocks[func.entry].row_count + 1):
horiz_edges[row] = [None] * (self.blocks[func.entry].col_count + 1)
vert_edges[row] = [None] * (self.blocks[func.entry].col_count + 1)
edge_valid[row] = [True] * (self.blocks[func.entry].col_count + 1)
for col in range(0, self.blocks[func.entry].col_count + 1):
horiz_edges[row][col] = []
vert_edges[row][col] = []
for block in self.blocks.values():
edge_valid[block.row][block.col + 1] = False
# Perform edge routing
for block in self.blocks.values():
start = block
for edge in block.block.exits:
end = self.blocks[edge]
color = Qt.black
if edge == block.block.true_path:
color = QColor(0, 144, 0)
elif edge == block.block.false_path:
color = QColor(144, 0, 0)
start.edges += [
self.routeEdge(horiz_edges, vert_edges, edge_valid, start,
end, color)
]
# Compute edge counts for each row and column
col_edge_count = [0] * (self.blocks[func.entry].col_count + 1)
row_edge_count = [0] * (self.blocks[func.entry].row_count + 1)
for row in range(0, self.blocks[func.entry].row_count + 1):
for col in range(0, self.blocks[func.entry].col_count + 1):
if len(horiz_edges[row][col]) > row_edge_count[row]:
row_edge_count[row] = len(horiz_edges[row][col])
if len(vert_edges[row][col]) > col_edge_count[col]:
col_edge_count[col] = len(vert_edges[row][col])
# Compute row and column sizes
col_width = [0] * (self.blocks[func.entry].col_count + 1)
row_height = [0] * (self.blocks[func.entry].row_count + 1)
for block in self.blocks.values():
if (int(block.width / 2)) > col_width[block.col]:
col_width[block.col] = int(block.width / 2)
if (int(block.width / 2)) > col_width[block.col + 1]:
col_width[block.col + 1] = int(block.width / 2)
if int(block.height) > row_height[block.row]:
row_height[block.row] = int(block.height)
# Compute row and column positions
col_x = [0] * self.blocks[func.entry].col_count
row_y = [0] * self.blocks[func.entry].row_count
self.col_edge_x = [0] * (self.blocks[func.entry].col_count + 1)
self.row_edge_y = [0] * (self.blocks[func.entry].row_count + 1)
x = 16
for i in range(0, self.blocks[func.entry].col_count):
self.col_edge_x[i] = x
x += 8 * col_edge_count[i]
col_x[i] = x
x += col_width[i]
y = 16
for i in range(0, self.blocks[func.entry].row_count):
self.row_edge_y[i] = y
y += 8 * row_edge_count[i]
row_y[i] = y
y += row_height[i]
self.col_edge_x[self.blocks[func.entry].col_count] = x
self.row_edge_y[self.blocks[func.entry].row_count] = y
self.width = x + 16 + (
8 * col_edge_count[self.blocks[func.entry].col_count])
self.height = y + 16 + (
8 * row_edge_count[self.blocks[func.entry].row_count])
# Compute node positions
for block in self.blocks.values():
block.x = int((col_x[block.col] + col_width[block.col] +
4 * col_edge_count[block.col + 1]) -
(block.width / 2))
if (block.x +
block.width) > (col_x[block.col] + col_width[block.col] +
col_width[block.col + 1] +
8 * col_edge_count[block.col + 1]):
block.x = int((col_x[block.col] + col_width[block.col] +
col_width[block.col + 1] +
8 * col_edge_count[block.col + 1]) -
block.width)
block.y = row_y[block.row]
# Precompute coordinates for edges
for block in self.blocks.values():
for edge in block.edges:
start = edge.points[0]
start_row = start[0]
start_col = start[1]
last_index = edge.start_index
last_pt = QPoint(
self.col_edge_x[start_col] + (8 * last_index) + 4,
block.y + block.height + 4 - (2 * self.charWidth))
pts = [last_pt]
for i in range(0, len(edge.points)):
end = edge.points[i]
end_row = end[0]
end_col = end[1]
last_index = end[2]
if start_col == end_col:
new_pt = QPoint(
last_pt.x(),
self.row_edge_y[end_row] + (8 * last_index) + 4)
else:
new_pt = QPoint(
self.col_edge_x[end_col] + (8 * last_index) + 4,
last_pt.y())
pts += [new_pt]
last_pt = new_pt
start_col = end_col
new_pt = QPoint(last_pt.x(), edge.dest.y + self.charWidth - 1)
pts += [new_pt]
edge.polyline = pts
pts = [
QPoint(new_pt.x() - 3,
new_pt.y() - 6),
QPoint(new_pt.x() + 3,
new_pt.y() - 6), new_pt
]
edge.arrow = pts
# Adjust scroll bars for new size
areaSize = self.viewport().size()
self.adjustSize(areaSize.width(), areaSize.height())
if self.desired_pos:
# There was a position saved, navigate to it
self.horizontalScrollBar().setValue(self.desired_pos[0])
self.verticalScrollBar().setValue(self.desired_pos[1])
elif self.cur_instr != None:
self.show_cur_instr()
else:
# Ensure start node is visible
start_x = self.blocks[func.entry].x + self.renderXOfs + int(
self.blocks[func.entry].width / 2)
self.horizontalScrollBar().setValue(start_x -
int(areaSize.width() / 2))
self.verticalScrollBar().setValue(0)
self.update_id = func.update_id
self.ready = True
self.viewport().update(0, 0, areaSize.width(), areaSize.height())
def updateTimerEvent(self):
status = self.analysis.status
if status != self.status:
self.status = status
self.statusUpdated.emit(self)
if self.function is None:
return
if self.ready:
# Check for updated code
self.analysis.lock.acquire()
if self.update_id != self.analysis.functions[
self.function].update_id:
self.renderFunction(self.analysis.functions[self.function])
self.analysis.lock.release()
return
# View not up to date, check to see if active function is ready
self.analysis.lock.acquire()
if self.analysis.functions.has_key(self.function):
if self.analysis.functions[self.function].ready:
# Active function now ready, generate graph
self.renderFunction(self.analysis.functions[self.function])
self.analysis.lock.release()
def show_cur_instr(self):
for block in self.blocks.values():
row = len(block.block.header_text.lines)
for instr in block.block.instrs:
if self.cur_instr == instr.addr:
x = block.x + int(block.width / 2)
y = block.y + (2 * self.charWidth) + int(
(row + 0.5) * self.charHeight)
self.horizontalScrollBar().setValue(
x + self.renderXOfs -
int(self.horizontalScrollBar().pageStep() / 2))
self.verticalScrollBar().setValue(
y + self.renderYOfs -
int(self.verticalScrollBar().pageStep() / 2))
return
row += len(instr.text.lines)
def navigate(self, addr):
# Check to see if address is within current function
for block in self.blocks.values():
row = len(block.block.header_text.lines)
for instr in block.block.instrs:
if (addr >= instr.addr) and (addr <
(instr.addr + len(instr.opcode))):
self.cur_instr = instr.addr
self.show_cur_instr()
self.viewport().update()
return True
row += len(instr.text.lines)
# Check other functions for this address
func, instr = self.analysis.find_instr(addr)
if func != None:
self.function = func
self.cur_instr = instr
self.highlight_token = None
self.ready = False
self.desired_pos = None
self.viewport().update()
return True
return False
def make_proc(self, addr):
# Create a procedure at the requested address if one does not already exist
if self.data.architecture() is None:
# Architecture not defined yet, ask the user and set it now
arch_dlg = ArchitectureDialog(self)
if arch_dlg.exec_() == QDialog.Rejected:
return False
self.data.default_arch = arch_dlg.result
self.analysis.lock.acquire()
if addr not in self.analysis.functions:
self.analysis.queue.append(addr)
self.analysis.lock.release()
self.function = addr
self.cur_instr = None
self.highlight_token = None
self.ready = False
self.desired_pos = None
self.viewport().update()
return True
def navigate_to_history_entry(self, entry):
self.function = entry.function
self.ready = False
self.desired_pos = [entry.scroll_x, entry.scroll_y]
self.cur_instr = entry.cur_instr
self.highlight_token = entry.highlight_token
self.viewport().update()
def get_history_entry(self):
return DisassemblerHistoryEntry(self)
def fontChanged(self):
self.initFont()
if self.ready:
# Rerender function to update layout
self.analysis.lock.acquire()
self.renderFunction(self.analysis.functions[self.function])
self.analysis.lock.release()
def getPriority(data, ext):
if Analysis.isPreferredForFile(data):
return 80
return 0
getPriority = staticmethod(getPriority)
def getViewName():
return "Disassembler"
getViewName = staticmethod(getViewName)
def getShortViewName():
return "Disassembler"
getShortViewName = staticmethod(getShortViewName)
def handlesNavigationType(name):
return (name == "disassembler") or (name == "make_proc")
handlesNavigationType = staticmethod(handlesNavigationType)
def main():
serial_number=''
path_root='../Tests'
offsets=''
test_date=''
date = str(datetime.datetime.now()).split(' ')[0]
for arg in sys.argv:
if arg.split('=')[0]=='serial_number':
serial_number=arg.split('=')[1]
if arg.split('=')[0]=='test_date':
test_date=arg.split('=')[1]
path=path_root+'/'+serial_number
offsets=''
with open(path+'/data/offsets.csv','r') as csvfile:
count=0
plots = csv.reader(csvfile, delimiter=',')
for row in plots:
if count > 0:
offsets+=', '+row[0]
else:
offsets+=row[0]
count+=1
Analysis.report='y'
Analysis.data='bias'
Analysis.path = path
Analysis.trial_id = '1'
Analysis.ave_time='1'
Analysis.num_points=15
Analysis.show_plot='n'
Analysis.save_plot='y'
bias_out=Analysis.analyze()
Analysis.report='y'
Analysis.data='dac'
Analysis.path = path
Analysis.trial_id = '1'
Analysis.ave_time='1'
Analysis.num_points=15
Analysis.show_plot='n'
Analysis.save_plot='y'
dac_out=Analysis.analyze()
Analysis.report='y'
Analysis.data='current'
Analysis.path = path
Analysis.trial_id = '1'
Analysis.ave_time='1'
Analysis.num_points=15
Analysis.show_plot='n'
Analysis.save_plot='y'
current_out_one=Analysis.analyze()
Analysis.report='y'
Analysis.data='current'
Analysis.path = path
Analysis.trial_id = '1'
Analysis.ave_time='100'
Analysis.num_points=15
Analysis.show_plot='n'
Analysis.save_plot='y'
current_out_hund=Analysis.analyze()
document=R'''
%\documentclass{revtex4-1}
\documentclass{article}%
%\usepackage{amsmath}%
%\usepackage{amsfonts}%
\usepackage{amssymb}%
\usepackage{graphicx,float}
\usepackage{amsmath}
\usepackage{geometry}
\usepackage{booktabs}
\usepackage{float}
\usepackage{makecell}
\geometry{letterpaper}
\begin{document}
\title{NSLS2\_EM Serial \#'''+serial_number+r''' Test Report}
\author{Kon Aoki}
%\\Colorado College}
\date{'''+date+r'''}
\maketitle
\section{Summary}
The NSLS2\_EM electrometer serial \#'''+serial_number+r''' was tested for its accuracy of readouts.
%PROCEDURE
\section{Procedure}
\begin{enumerate}
\item Record ADC offsets
\item Grounding
\begin{enumerate}
\item Confirm chassis ground to mains ground
\item Confirm outer conductor of coax connector grounded to chassis
\end{enumerate}
\item Bias (Test date: '''+test_date+r''')
\begin{enumerate}
\item Set bias via EPICS \label{itm:1s}
\item Measure and record bias output
\item Repeat measurement 3 times \label{itm:1l}
\item Repeat procedure \ref{itm:1s}-\ref{itm:1l} for biases -10 to 10V in 1V increments
\end{enumerate}
\item DAC Output (Test date: '''+test_date+r''')
\begin{enumerate}
\item Set DAC via EPICS \label{itm:2s}
\item Measure and record DAC output \label{itm:2s2}
\item Repeat measurement 3 times \label{itm:2l}
\item Repeat procedure \ref{itm:2s}-\ref{itm:2l} for outputs -10 to 10V in 1V increments \label{itm:l2}
\item Repeat all steps for each channel
\end{enumerate}
\item Current measurement at 1ms averaging time (Test date: '''+test_date+r''')
\begin{enumerate}
\item Set values per read to 50
\item Set averaging time to 1ms
\item Set number of points to collect to 100
\item Set ADC offset
\item Set range \label{itm:3s}
\item Set input current \label{itm:3s2}
\item Wait 5.1 seconds until data is ready (averaging time $\times$ number of points $+$ $5$s buffer)
\item Record current measurement \label{itm:3l}
\item Repeat procedure \ref{itm:3s2}-\ref{itm:3l} for 15 inputs between 0 and 20\% above the range setting
\item Repeat procedure \ref{itm:3s}-\ref{itm:3l} for ranges 1$\mu$A to 50mA
\item Repeat all steps for each channel
\end{enumerate}
\item Current measurement at 100ms averaging time (Test date: '''+test_date+r''')
\begin{enumerate}
\item Set values per read to 50
\item Set averaging time to 100ms
\item Set number of points to collect to 100
\item Set ADC offset
\item Set range \label{itm:4s}
\item Set input current \label{itm:4s2}
\item Wait 15 seconds until data is ready (averaging time $\times$ number of points $+$ $5$s buffer)
\item Record current measurement \label{itm:4l}
\item Repeat procedure \ref{itm:4s2}-\ref{itm:4l} for 15 inputs between 0 and 20\% above the range setting
\item Repeat procedure \ref{itm:4s}-\ref{itm:4l} for ranges 1$\mu$A to 50mA
\item Repeat all steps for each channel
\end{enumerate}
\end{enumerate}
%RESULTS
\section{Results}
ADC Offsets were set at '''+offsets+r'''. \\
Confirmed chassis ground to mains ground\\
Confirmed outer conductor of coax connector grounded to chassis.
%Bias
\subsection{Bias Voltage}
\begin{figure}[H]
\centering
\includegraphics[width=1\textwidth]{plot/bias.png}
\caption{Bias Voltage}
\label{fig_bias}
\end{figure}
\begin{table}[H]
\centering
\begin{tabular}{|l|l|l|}
\hline
Slope & Offset (V) & Median STD (V)\\ \hline
'''+bias_out+r'''
\end{tabular}
\end{table}
%DAC
\subsection{DAC Output}
\begin{figure}[H]
\centering
\includegraphics[width=1\textwidth]{plot/dac.png}
\caption{DAC Output}
\label{fig_dac}
\end{figure}
\begin{table}[H]
\centering
\begin{tabular}{|l|l|l|l|}
\hline
Channel & Slope & Offset (V) & Median STD (V)\\ \hline
'''+dac_out+r'''
\end{tabular}
\end{table}
%CURRENT MEASUREMENT 1ms
\subsection{Current Measurement 1ms Averaging Time}
*Note that measurements taken from input values above the range setting was ignored for the model for ranges 1$\mu$A to 1mA. For the 50mA range, input values above 40mA (80\%) was ignored for the model.
\begin{figure}[H]
\centering
\includegraphics[width=0.55\textwidth]{plot/1ms_range1.png}
\caption{Range 1$\mu$A}
\label{fig_adc}
\end{figure}
\begin{figure}[H]
\centering
\includegraphics[width=0.55\textwidth]{plot/1ms_range10.png}
\caption{Range 10$\mu$A}
\label{fig_adc2}
\end{figure}
\begin{figure}[H]
\centering
\includegraphics[width=0.55\textwidth]{plot/1ms_range100.png}
\caption{Range 100$\mu$A}
\label{fig_adc3}
\end{figure}
\begin{figure}[H]
\centering
\includegraphics[width=0.55\textwidth]{plot/1ms_range1000.png}
\caption{Range 1mA}
\label{fig_adc4}
\end{figure}
\begin{figure}[H]
\centering
\includegraphics[width=0.55\textwidth]{plot/1ms_range50000.png}
\caption{Range 50mA}
\label{fig_adc5}
\end{figure}
\begin{table}[H]
\centering
\begin{tabular}{|l|l|l|l|l|}
\hline
Channel & Range ($\mu$A) & Slope & Offset ($\mu$A) & Median STD ($\mu$A)\\ \Xhline{3\arrayrulewidth}
'''+current_out_one+r'''
\end{tabular}
\end{table}
%CURRENT MEASUREMENT 100ms
\subsection{Current Measurement 100ms Averaging Time}
*Note that measurements taken from input values above the range setting was ignored for the model for ranges 1$\mu$A to 1mA. For the 50mA range, input values above 40mA (80\%) was ignored for the model.
\begin{figure}[H]
\centering
\includegraphics[width=0.55\textwidth]{plot/100ms_range1.png}
\caption{Range 1$\mu$A}
\label{fig_adc}
\end{figure}
\begin{figure}[H]
\centering
\includegraphics[width=0.55\textwidth]{plot/100ms_range10.png}
\caption{Range 10$\mu$A}
\label{fig_adc2}
\end{figure}
\begin{figure}[H]
\centering
\includegraphics[width=0.55\textwidth]{plot/100ms_range100.png}
\caption{Range 100$\mu$A}
\label{fig_adc3}
\end{figure}
\begin{figure}[H]
\centering
\includegraphics[width=0.55\textwidth]{plot/100ms_range1000.png}
\caption{Range 1mA}
\label{fig_adc4}
\end{figure}
\begin{figure}[H]
\centering
\includegraphics[width=0.55\textwidth]{plot/100ms_range50000.png}
\caption{Range 50mA}
\label{fig_adc5}
\end{figure}
\begin{table}[H]
\centering
\begin{tabular}{|l|l|l|l|l|}
\hline
Channel & Range ($\mu$A) & Slope & Offset ($\mu$A) & Median STD ($\mu$A)\\ \Xhline{3\arrayrulewidth}
'''+current_out_hund+r'''
\end{tabular}
\end{table}
\end{document}
'''
report_path=path+"/Unit"+serial_number+"_report.tex"
print("Report generated at "+report_path)
f = open(report_path,"w")
f.write(document)
f.close()
def main(experiment, experimentName, n=1):
print("Started Runs")
start: float = timeit.default_timer()
finishedRuns = []
numberOfIterations = list(range(n))
byTarget = {"makespan": {}, "avgFlowTime": {}, "maximumLateness": {}}
for target in byTarget:
for sf in experiment["schedulers"]:
byTarget[target][sf] = []
if Path(experimentName + ".csv").is_file():
with open(experimentName + ".csv", newline="") as csvfile:
reader = csv.reader(csvfile, delimiter=";")
for row in reader:
byTarget[row[0]][row[1]].append((float(row[2]), float(row[3])))
#print(", ".join(row))
product = itertools.product(*(list(experiment.values()))[:-2])
for conf in product: #itertools.product(numberOfJobs,numberOfNodes,seqR,largeR,timespan,minSeq,maxSeq,minPar,maxPar):
for i in numberOfIterations:
jobs: List[Simulation.Job] = Generator.generate(*conf)
for sf in experiment["schedulers"]:
sys: Simulation.System = Simulation.System(
jobs.copy(), conf[1], schedulerConverter[sf])
finished: List[Simulation.Job] = sys.run()
analysis = Analysis.standardAnalysis(finished)
for target in byTarget:
byTarget[target][sf].append(
(conf[xValueConverter[experiment["x-axis"][0]]],
analysis[target]))
#print (byTarget)
#sortieren nach x
#dbCopy = {**bySchedulers}
#save to file:
with open(experimentName + ".csv", "w", newline="") as csvfile:
writer = csv.writer(csvfile, delimiter=";")
for target in byTarget:
for sf in byTarget[target]:
for valuePair in byTarget[target][sf]:
writer.writerow([target] + [sf] + list(valuePair))
fig, axs = plt.subplots(3)
i = 0
for targetFunktion in ["makespan", "avgFlowTime", "maximumLateness"]:
#setup
xLabel = experiment["x-axis"][0]
yLabel = targetFunktion
for sf in byTarget[targetFunktion]:
xs = []
ys = []
pairs = sorted(byTarget[targetFunktion][sf], key=lambda x: x[0])
grouped = itertools.groupby(pairs, lambda x: x[0])
for key, group in grouped:
vals = list(group)
xs.append(key)
ys.append(sum(map(lambda x: x[1], vals)) / len(vals))
axs[i].plot(xs, ys, label=sf)
axs[i].set_ylabel(targetFunktion)
axs[i].set_ylim(ymin=0)
axs[i].legend(bbox_to_anchor=(1.0, 1),
loc="upper left",
fontsize="xx-small")
i += 1
plt.xlabel(experiment["x-axis"][0])
plt.show()
def threadCrown(filepath):
global io
rtpSkel=-1
crownT=OrderedDict()
imgL=[]
stemCorrection=bool(int(options[8][1]))
print io.getHomePath()
oldHome=io.getHomePath()
os.chdir(io.getHomePath())
io.setHomePath('./Crown/')
f=io.scanDir()
for (counter,i) in enumerate(f):
io.setFileName(os.path.basename(i))
io.setidIdx(imgID)
print 'processing Crown file: '+i
xScale=allPara[counter][7]
yScale=allPara[counter][8]
analysis=Analysis.Analysis(io,(xScale+yScale)/2)
rtp=RootTipPaths.RootTipPaths(io)
try:
img=scipy.misc.imread(i,flatten=True)
except:
print 'Image not readable'
img=-1
if len(img)>0:
seg=Segmentation.Segmentation(img,io)
imgL=seg.label()
print 'compute root profile'
currT=time.time()
if ifAnyKeyIsTrue(['AVG_DENSITY','WIDTH_MED','WIDTH_MAX','DIA_STM_SIMPLE','D10','D20','D30','D40','D50','D60','D70','D80','D90','DS10','DS20','DS30','DS40','DS50','DS60','DS70','DS80','DS90','AREA','ANG_TOP','ANG_BTM']):
crownT['AVG_DENSITY'],crownT['WIDTH_MED'],crownT['WIDTH_MAX'],crownT['D10'],crownT['D20'],crownT['D30'],crownT['D40'],crownT['D50'],crownT['D60'],crownT['D70'],crownT['D80'],crownT['D90'],crownT['DS10'],crownT['DS20'],crownT['DS30'],crownT['DS40'],crownT['DS50'],crownT['DS60'],crownT['DS70'],crownT['DS80'],crownT['DS90'],crownT['AREA'],crownT['DIA_STM_SIMPLE'],crownT['ANG_TOP'],crownT['ANG_BTM']=analysis.getWidthOverHeight(imgL,xScale,yScale)
print 'Mask traits computed '+str(time.time()-currT)+'s'
if ifAnyKeyIsTrue(['DIA_STM','TD_MED','TD_AVG','STA_RANGE','STA_DOM_I','STA_DOM_II','STA_25_I','STA_25_II','STA_50_I','STA_50_II','STA_75_I','STA_75_II','STA_90_I','STA_90_II','RTA_DOM_I','RTA_DOM_II','STA_MIN','STA_MAX','STA_MED','RTA_RANGE','RTA_MIN','RTA_MAX','RTA_MED','NR_RTP_SEG_I','NR_RTP_SEG_II','ADVT_COUNT','BASAL_COUNT','ADVT_ANG','BASAL_ANG','HYP_DIA','TAP_DIA','MAX_DIA_90','DROP_50','CP_DIA25','CP_DIA50','CP_DIA75','CP_DIA90','SKL_DEPTH','SKL_WIDTH']):
currT=time.time()
skel=Skeleton.Skeleton(imgL)
testSkel,testDia=skel.skel(imgL)
print 'Medial axis computed '+str(time.time()-currT)+'s'
currT=time.time()
path,skelGraph,crownT['DIA_STM'],skelSize=seg.findThickestPath(testSkel,testDia,xScale,yScale)
allPara[counter][10]=skelSize
print 'Central path computed '+str(time.time()-currT)+'s'
if ifAnyKeyIsTrue(['TD_MED','TD_AVG','STA_RANGE','STA_DOM_I','STA_DOM_II','STA_25_I','STA_25_II','STA_50_I','STA_50_II','STA_75_I','STA_75_II','STA_90_I','STA_90_II','RTA_DOM_I','RTA_DOM_II','STA_MIN','STA_MAX','STA_MED','RTA_RANGE','RTA_MIN','RTA_MAX','RTA_MED','NR_RTP_SEG_I','NR_RTP_SEG_II','ADVT_COUNT','BASAL_COUNT','ADVT_ANG','BASAL_ANG','HYP_DIA','TAP_DIA','MAX_DIA_90','DROP_50','CP_DIA25','CP_DIA50','CP_DIA75','CP_DIA90','SKL_DEPTH','SKL_WIDTH','RTP_COUNT']):
print 'Compute RTP skeleton'
currT=time.time()
rtpSkel,crownT['RTP_COUNT'], crownT['TD_MED'],crownT['TD_AVG'],crownT['MAX_DIA_90'], rtps, tips, crownT['SKL_WIDTH'], crownT['SKL_DEPTH'] =rtp.getRTPSkeleton(path,skelGraph,True)
seg.setTips(tips)
print 'RTP Skeleton computed '+str(time.time()-currT)+'s'
allPara[len(allPara)-1][2]=seg.getFail()
if ifAnyKeyIsTrue(['RDISTR_X','RDISTR_Y']):
print 'Compute spatial root distribution'
currT=time.time()
crownT['RDISTR_X'],crownT['RDISTR_Y']=analysis.getSymmetry(rtps,rtpSkel)
print 'Symmetry computed '+str(time.time()-currT)+'s'
if rtpSkel!=-1:
if ifAnyKeyIsTrue(['NR_RTP_SEG_I','NR_RTP_SEG_II','ADVT_COUNT','BASAL_COUNT','ADVT_ANG','BASAL_ANG','HYP_DIA','TAP_DIA']):
print 'searching for hypocotyl'
currT=time.time()
branchRad,nrPaths=seg.findHypocotylCluster(path,rtpSkel)
print 'hypocotyl computed '+str(time.time()-currT)+'s'
print 'starting kmeans'
try:
currT=time.time()
c1x,c1y,c2x,c2y = analysis.plotDiaRadius(nrPaths, branchRad,path,2)
print '2 clusters computed in '+str(time.time()-currT)+'s'
currT=time.time()
segImg=seg.makeSegmentationPicture(path,rtpSkel,img,xScale,yScale,c1x,c1y,c2x,c2y)
scipy.misc.imsave(io.getHomePath()+'/Result/' +io.getFileName()+ 'Seg2.png', segImg)
crownT['ADVT_COUNT'],crownT['BASAL_COUNT'],crownT['NR_RTP_SEG_I'],crownT['NR_RTP_SEG_II'], crownT['HYP_DIA'], crownT['TAP_DIA'] =analysis.countRootsPerSegment(c1y,c2y,c1x,c2x)
except:
c1x=None
c1y=None
c2x=None
c2y=None
pass
crownT['DROP_50']=analysis.RTPsOverDepth(path,rtpSkel)
print 'count roots per segment'
print 'Root classes computed in '+str(time.time()-currT)+'s'
if ifAnyKeyIsTrue(['ADVT_ANG','BASAL_ANG','STA_RANGE','STA_DOM_I','STA_DOM_II','STA_25_I','STA_25_II','STA_50_I','STA_50_II','STA_75_I','STA_75_II','STA_90_I','STA_90_II','RTA_DOM_I','RTA_DOM_II','STA_MIN','STA_MAX','STA_MED','RTA_RANGE','RTA_MIN','RTA_MAX','RTA_MED']):
currT=time.time()
lat,corrBranchpts=seg.findLaterals(rtps, rtpSkel,(xScale+yScale)/2, None)
print 'seg.findLaterals computed in '+str(time.time()-currT)+'s'
print 'Compute angles at 2cm'
currT=time.time()
if c1x!=None and c1y!=None and c2x!=None and c2y!=None: crownT['ADVT_ANG'],crownT['BASAL_ANG']=analysis.anglesPerClusterAtDist(c1y, c2y, rtpSkel, path, lat, corrBranchpts, (xScale+yScale)/2, dist=20)
else:
crownT['ADVT_ANG']='nan'
crownT['BASAL_NG']='nan'
print 'angles at 2cm computed in '+str(time.time()-currT)+'s'
if ifAnyKeyIsTrue(['STA_25_I','STA_25_II','STA_50_I','STA_50_II','STA_75_I','STA_75_II','STA_90_I','STA_90_II']):
try:
print 'compute quantile angles'
currT=time.time()
a25,a50,a75,a90=analysis.calculateAngleQuantiles(path,lat,corrBranchpts,rtpSkel)
print 'angles computed in '+str(time.time()-currT)+'s'
except:
a25=['nan']
a50=['nan']
a75=['nan']
a90=['nan']
print 'ERROR: No quantile angles calculated'
if ifAnyKeyIsTrue(['RTA_RANGE','RTA_MIN','RTA_MAX','RTA_MED']):
try:
print 'compute angles'
currT=time.time()
crownT['RTA_MED'],crownT['RTA_MIN'],crownT['RTA_MAX'],crownT['RTA_RANGE'],anglesN=analysis.calculateAngles(path,lat,corrBranchpts,rtpSkel)
print 'RTA angle characteristics computed in '+str(time.time()-currT)+'s'
except:
print 'ERROR: No RTA angles calculated'
if ifAnyKeyIsTrue(['STA_RANGE','STA_MIN','STA_MAX','STA_MED']):
try:
print 'compute STA angles'
currT=time.time()
crownT['STA_RANGE'],crownT['STA_MED'],crownT['STA_MIN'],crownT['STA_MAX'],angles=analysis.getLateralAngles(path,lat,corrBranchpts,rtpSkel)
print 'STA angles characteristics computed in '+str(time.time()-currT)+'s'
except:
print 'ERROR: No STA angles calculated'
if ifAnyKeyIsTrue(['CP_DIA25','CP_DIA50','CP_DIA75','CP_DIA90']):
try:
print 'compute diameter quantils'
currT=time.time()
crownT['CP_DIA25'],crownT['CP_DIA50'],crownT['CP_DIA75'],crownT['CP_DIA90']=analysis.getDiameterQuantilesAlongSinglePath(path,rtpSkel)
print 'Tap diameters computed in '+str(time.time()-currT)+'s'
except:
print 'ERROR: No quantile diameters calculated'
if ifAnyKeyIsTrue(['STA_DOM_I','STA_DOM_II']):
try:
print 'compute STA dominant angles'
currT=time.time()
crownT['STA_DOM_I'],crownT['STA_DOM_II']=analysis.findHistoPeaks(angles)
print 'STA dominant angles computed in '+str(time.time()-currT)+'s'
except:
print 'ERROR: No dominant angles calculated (STA)'
if ifAnyKeyIsTrue(['STA_25_I','STA_25_II']):
try:
currT=time.time()
crownT['STA_25_I'],crownT['STA_25_II']=analysis.findHistoPeaks(a25)
print 'STA 25 angles computed in '+str(time.time()-currT)+'s'
except:
print 'ERROR: No dominant angles25 calculated'
if ifAnyKeyIsTrue(['STA_50_I','STA_50_II']):
try:
currT=time.time()
crownT['STA_50_I'],crownT['STA_50_II']=analysis.findHistoPeaks(a50)
print 'STA 50 angles computed in '+str(time.time()-currT)+'s'
except:
print 'ERROR: No dominant angles50 calculated'
if ifAnyKeyIsTrue(['STA_75_I','STA_75_II']):
try:
currT=time.time()
crownT['STA_75_I'],crownT['STA_75_II']=analysis.findHistoPeaks(a75)
print 'STA 75 angles computed in '+str(time.time()-currT)+'s'
except:
print 'ERROR: No dominant angles75 calculated'
if ifAnyKeyIsTrue(['STA_90_I','STA_90_II']):
try:
currT=time.time()
crownT['STA_90_I'],crownT['STA_90_II']=analysis.findHistoPeaks(a90)
print 'STA 90 angles computed in '+str(time.time()-currT)+'s'
except:
print 'ERROR: No dominant angles90 calculated'
if ifAnyKeyIsTrue(['RTA_DOM_I','RTA_DOM_II']):
try:
currT=time.time()
crownT['RTA_DOM_I'],crownT['RTA_DOM_II']=analysis.findHistoPeaks(anglesN)
print 'angles computed in '+str(time.time()-currT)+'s'
except:
print 'ERROR: No dominant RTA angles calculated'
io.setHomePath(oldHome)
if maxExRoot >= 1:
rtpSkel=-1
os.chdir(io.getHomePath())
io.setHomePath('./Lateral/')
f=io.scanDir()
for (counter,i) in enumerate(f):
print 'processing lateral file: '+i
if maxExRoot>0:
xScale=allPara[counter/maxExRoot][7]
yScale=allPara[counter/maxExRoot][8]
io.setFileName(os.path.basename(i))
else:
xScale=allPara[counter][7]
yScale=allPara[counter][8]
io.setFileName(os.path.basename(i))
io.setidIdx(counter)
rtp=RootTipPaths.RootTipPaths(io)
analysis=Analysis.Analysis(io,(xScale+yScale)/2)
try:
img=scipy.misc.imread(i,flatten=True)
except:
print 'Image not readable'
img=[]
pass
if len(img)>0:
seg=Segmentation.Segmentation(img,io=io)
imgL=seg.label()
if imgL!=None:
skel=Skeleton.Skeleton(imgL)
testSkel,testDia=skel.skel(imgL)
path,skelGraph=seg.findThickestPathLateral(testSkel,testDia,xScale,yScale)
if ifAnyKeyIsTrue(['LT_AVG_LEN','NODAL_LEN','LT_BRA_FRQ','NODAL_AVG_DIA','LT_AVG_ANG','LT_ANG_RANGE','LT_MIN_ANG','LT_MAX_ANG','LT_DIST_FIRST','LT_MED_DIA','LT_AVG_DIA']):
rtpSkel,_,crownT['LT_MED_DIA'],crownT['LT_AVG_DIA'],_,rtps,_,_,_=rtp.getRTPSkeleton(path,skelGraph,True)
if rtpSkel!=-1:
if ifAnyKeyIsTrue(['LT_BRA_FRQ']):
crownT['LT_BRA_FRQ']=analysis.getBranchingfrequencyAlongSinglePath(rtps,path)
crownT['NODAL_AVG_DIA'],_=analysis.getDiametersAlongSinglePath(path,rtpSkel,(xScale+yScale)/2)
crownT['NODAL_LEN']=analysis.getLengthOfPath(path)
if ifAnyKeyIsTrue(['LT_DIST_FIRST','LT_AVG_LEN','LT_BRA_FRQ','LT_ANG_RANGE','LT_AVG_ANG','LT_MIN_ANG','LT_MAX_ANG']):
lat,corrBranchpts,crownT['LT_DIST_FIRST']=seg.findLaterals(rtps, rtpSkel,(xScale+yScale)/2,path)
if ifAnyKeyIsTrue(['LT_AVG_LEN']):
crownT['LT_AVG_LEN']=analysis.getLateralLength(lat,path,rtpSkel)
if ifAnyKeyIsTrue(['LT_ANG_RANGE','LT_AVG_ANG','LT_MIN_ANG','LT_MAX_ANG']):
crownT['LT_ANG_RANGE'],crownT['LT_AVG_ANG'],crownT['LT_MIN_ANG'],crownT['LT_MAX_ANG'],_=analysis.getLateralAngles(path,lat,corrBranchpts,rtpSkel)
allCrown.append(crownT.copy())
else:
allCrown.append(crownT.copy())
io.setHomePath(oldHome)
def DiffusionConstants(self):
with Parallel(n_jobs=self.threads) as parallel:
self.LDValues = {}
for key in self.bacteria.bacterium.keys():
self.LDValues[key] = Analysis.LDValues(
self.bacteria.config[key])
# %% Linear - LogLog fullscale
gp.c('reset')
gp.c('set logscale xy 10')
gp.c('set xlabel "{/Symbol t} (s)"')
gp.c('set ylabel "MSD (m^2)"')
gp.c('set key top left')
gp.c("set terminal pngcairo enhanced" +
" size 1600,1200 font 'ariel, 14'")
amalg_dat_name = []
amalg_titles = []
for key in self.bacteria.bacterium.keys():
key_title = self.bacteria.config[key].name
print('Started: %s \t Linear Analysis' % (key_title))
output = os.path.join(self.graph_dir, '%s_linear.png' % (key))
gp.c('set output "%s"' % (output))
g_title = 'Analysis of Linear Mean Squared Displacement - %s'\
% (key_title)
gp.c('set title "%s"' % (g_title))
tau = Analysis.TauCalc(self.bacteria.config[key])
gp.c('set xrange [%f:%f]' %
(tau.min() * 0.75, tau.max() * 1.25))
results_array = parallel(
delayed(Analysis.Linear)(self.bacteria.bacterium[key]
[bact], self.bacteria.config[key])
for bact in self.bacteria.bacterium[key].keys())
size = len(results_array)
dat_name = os.path.join(self.plot_dir,
'%s_msd_lin.dat' % (key))
graph_out = np.vstack((tau, results_array))
gp.s(graph_out, dat_name)
title = ['notitle' for i in range(size)]
plot_string = plotStringSingleFile(size, dat_name,
'with points', title)
gp.c(plot_string)
output = os.path.join(self.graph_dir,
'%s_linear_mean.png' % (key))
gp.c('set output "%s"' % (output))
mean_results = np.mean(results_array, axis=0)
std_dev = np.std(results_array, axis=0)
std_error = std_dev / np.sqrt(size)
current_results = np.vstack((tau, mean_results, std_error))
dat_name = os.path.join(self.plot_dir,
'%s_msd_lin_mean.dat' % (key))
gp.s(current_results, dat_name)
plot_string = 'plot "%s" u 1:2:3 with yerrorbars' % (dat_name)
plot_string = plot_string + ' title "Mean Linear MSD"'
gp.c(plot_string)
amalg_dat_name.append(dat_name)
amalg_titles.append('title "%s"' % (key_title))
print('Completed %s \t Linear Analysis' % (key))
amalg_formatting = 'u 1:2:3 with yerrorlines'
amalg_plot_string = plotStringMultiFile(len(amalg_dat_name),
amalg_dat_name,
amalg_formatting,
amalg_titles)
output = os.path.join(self.graph_dir, 'linear_mean_amalg.png')
gp.c('set output "%s"' % (output))
g_title = 'Analysis of Linear Mean Squared Displacement'
gp.c('set title "%s"' % (g_title))
gp.c('set xrange [*:*]')
gp.c(amalg_plot_string)
# %% Linear - High Range (>1 sec)
gp.c('reset')
gp.c('set xlabel "{/Symbol t} (s)"')
gp.c('set ylabel "MSD (m^2)"')
gp.c('set key top left')
gp.c("set terminal pngcairo enhanced" +
" size 1600,1200 font 'ariel, 14'")
amalg_dat_name = []
amalg_titles = []
amalg_grad = []
amalg_grad_titles = []
for key in self.bacteria.bacterium.keys():
line_colour = 1
key_title = self.bacteria.config[key].name
print('Started: %s \t Linear Analysis High Range' %
(key_title))
output = os.path.join(self.graph_dir,
'%s_linear_hr.png' % (key))
gp.c('set output "%s"' % (output))
g_title = 'Analysis of Linear Mean Squared Displacement %s'\
% (key_title)
gp.c('set title "%s"' % (g_title))
tau = Analysis.TauCalcHR(self.bacteria.config[key])
gp.c('set xrange [%f:%f]' % (tau.min() - 5, tau.max() + 5))
results_array = parallel(
delayed(Analysis.LinearHighRange)
(self.bacteria.bacterium[key][bact],
self.bacteria.config[key])
for bact in self.bacteria.bacterium[key].keys())
size = len(results_array)
dat_name = os.path.join(self.plot_dir,
'%s_msd_lin_hr.dat' % (key))
graph_out = np.vstack((tau, results_array))
gp.s(graph_out, dat_name)
title = ['notitle' for i in range(size)]
plot_string = plotStringSingleFile(size, dat_name,
'with points', title)
gp.c(plot_string)
output = os.path.join(self.graph_dir,
'%s_linear_mean_hr.png' % (key))
gp.c('set output "%s"' % (output))
mean_results = np.mean(results_array, axis=0)
std_dev = np.std(results_array, axis=0)
std_error = std_dev / np.sqrt(size)
current_results = np.vstack((tau, mean_results, std_error))
gradient, y_intercept, r_value, p_value, grad_err\
= linregress(tau, mean_results)
exp_diff = "Gradient = %.5e {/Symbol \261} %.5e" % (gradient,
grad_err)
r_2 = "R^2 = %f" % (r_value**2)
fit_title = ("%s, %s, %s") % (key_title, exp_diff, r_2)
dat_name = os.path.join(self.plot_dir,
'%s_msd_lin_mean_hr.dat' % (key))
gp.s(current_results, dat_name)
plot_string = 'plot "%s" u 1:2:3 with yerrorbars lc %d' %\
(dat_name, line_colour)
plot_string = plot_string + ' title "%s - Mean MSD"'\
% (key_title)
plot_string = plot_string + ', %e*x lc %d' %\
(gradient, line_colour)
plot_string = plot_string + 'title "%s"' % (fit_title)
gp.c(plot_string)
amalg_dat_name.append(dat_name)
amalg_titles.append('title "%s - Mean MSD" ' % (key_title))
amalg_grad.append(gradient)
amalg_grad_titles.append(fit_title)
print('Completed %s \t Linear Analysis High Range' %
(key_title))
amalg_formatting = 'u 1:2:3 with yerrorbars'
amalg_plot_string = plotStringMultiFileWithFit(
len(amalg_dat_name), amalg_dat_name, amalg_formatting,
amalg_titles, amalg_grad, amalg_grad_titles)
output = os.path.join(self.graph_dir, 'linear_mean_amalg_hr.png')
gp.c('set output "%s"' % (output))
g_title = 'Analysis of Linear Mean Squared Displacement'
gp.c('set title "%s"' % (g_title))
gp.c('set xrange [*:*]')
gp.c(amalg_plot_string)
# %% Rotational Analysis
amalg_dat_name = []
amalg_titles = []
gp.c('reset')
gp.c('set logscale xy 10')
gp.c('set xlabel "{/Symbol t} (s)"')
gp.c('set ylabel "MSD ({/Symbol q}^2)"')
gp.c('set key top left')
gp.c("set terminal pngcairo enhanced" +
" size 1600,1200 font 'ariel, 14'")
for key in self.bacteria.bacterium.keys():
key_title = self.bacteria.config[key].name
print("Started %s \t Rotational Analysis" % (key))
output = os.path.join(self.graph_dir,
'%s_rotational.png' % (key))
gp.c('set output "%s"' % (output))
g_title = 'Analysis of Rotational Mean Squared '\
+ 'Displacement - %s' % (key)
gp.c('set title "%s" noenhanced' % (g_title))
tau = Analysis.TauCalc(self.bacteria.config[key])
gp.c('set xrange [%f:%f]' %
(tau.min() * 0.75, tau.max() * 1.25))
results_array = parallel(
delayed(Analysis.Rotational)
(self.bacteria.bacterium[key][bact],
self.bacteria.config[key])
for bact in self.bacteria.bacterium[key].keys())
size = len(results_array)
dat_name = os.path.join(self.plot_dir,
'%s_msd_rot.dat' % (key))
graph_out = np.vstack((tau, results_array))
gp.s(graph_out, dat_name)
title = ['notitle' for i in range(size)]
plot_string = plotStringSingleFile(size, dat_name,
'with points', title)
gp.c(plot_string)
output = os.path.join(self.graph_dir,
'%s_rotational_mean.png' % (key))
gp.c('set output "%s"' % (output))
mean_results = np.mean(results_array, axis=0)
std_dev = np.std(results_array, axis=0)
std_error = std_dev / np.sqrt(size)
current_results = np.vstack((tau, mean_results, std_error))
dat_name = os.path.join(self.plot_dir,
'%s_msd_rot_mean.dat' % (key))
gp.s(current_results, dat_name)
plot_string = 'plot "%s" u 1:2:3 with yerrorbars' % (dat_name)
plot_string = plot_string + ' title "Mean Rotational MSD"'
gp.c(plot_string)
amalg_dat_name.append(dat_name)
amalg_titles.append('title "%s"' % (key_title))
print("Completed %s \t Rotational Analysis" % (key))
amalg_formatting = 'u 1:2:3 with yerrorlines'
amalg_plot_string = plotStringMultiFile(len(amalg_dat_name),
amalg_dat_name,
amalg_formatting,
amalg_titles)
output = os.path.join(self.graph_dir, 'rotational_mean_amalg.png')
gp.c('set output "%s"' % (output))
g_title = 'Analysis of Rotational Mean Squared Displacement'
gp.c('set title "%s"' % (g_title))
gp.c('set xrange [*:*]')
gp.c(amalg_plot_string)
# %%
gp.c('reset')
gp.c('set key top left')
gp.c("set terminal pngcairo enhanced" +
" size 1600,1200 font 'ariel, 14'")
gp.c('unset logscale')
gp.c('set ylabel "Probability Density"')
gp.c('set xlabel "Run to Run Angle (degrees)"')
amalg_dat_name = []
amalg_titles = []
for key in self.bacteria.bacterium.keys():
key_title = self.bacteria.config[key].name
print("Started %s \t Run to Run angles" % (key))
if not self.bacteria.config[key].run_behaviour:
print("%s is non-motile, ignoring" % (key))
continue
if self.bacteria.config[key].archaea_mode:
print("%s is an archaea, ignoring" % (key))
continue
output = os.path.join(self.graph_dir,
'%s_run_run_angle.png' % (key))
gp.c('set output "%s"' % (output))
title = 'Analysis of Run to Run Angle - %s'\
% (key)
gp.c('set title "%s"' % (title))
angle_list = parallel(
delayed(Analysis.RunRunAngles)(self.bacteria.cosines[key]
[bact])
for bact in self.bacteria.cosines[key].keys())
angle_array = []
for i in range(len(angle_list)):
angle_array = np.append(angle_array, angle_list[i])
angle_bins = np.linspace(0, np.pi, 40)
angle_mean = np.mean(angle_array)
angle_std = np.std(angle_array)
angle_std_err = angle_std / np.sqrt(len(angle_array))
self.LDValues[key].avg_tumble = angle_mean
self.LDValues[key].tumble_err = angle_std_err
angle_med = np.median(angle_array)
results, bin_edges = np.histogram(angle_array,
bins=angle_bins,
density=True)
angle_points = np.delete(angle_bins,
-1) + np.diff(angle_bins) / 2
graph_out = np.vstack((angle_points, results))
title = "%s, Mean = %6.2f, std. dev. = %6.2f, Median = %6.2f"\
% (key_title, angle_mean, angle_std, angle_med)
dat_name = os.path.join(self.plot_dir,
'%s_run_hist.dat' % (key))
gp.s(graph_out, dat_name)
plot_string = 'plot "%s" u 1:2 with points title "%s"'\
% (dat_name, title)
gp.c(plot_string)
amalg_dat_name.append(dat_name)
amalg_titles.append('title "%s"' % (title))
print("Completed %s \t Run to Run angles" % (key))
amalg_formatting = 'u 1:2 with points'
if amalg_dat_name:
amalg_plot_string = plotStringMultiFile(
len(amalg_dat_name), amalg_dat_name, amalg_formatting,
amalg_titles)
output = os.path.join(self.graph_dir,
'run_run_angle_amalg.png')
gp.c('set output "%s"' % (output))
g_title = 'Analysis of Run to Run Angles'
gp.c('set title "%s"' % (g_title))
gp.c('set xrange [*:*]')
gp.c(amalg_plot_string)
# %%
gp.c('reset')
gp.c('set key top left')
gp.c("set terminal pngcairo enhanced" +
" size 1600,1200 font 'ariel, 14'")
gp.c('set logscale y')
gp.c('set ylabel "Probability Density"')
gp.c('set xlabel "Run Duration (s)"')
gp.c('set xrange [*:*]')
amalg_dat_name = []
amalg_titles = []
for key in self.bacteria.bacterium.keys():
key_title = self.bacteria.config[key].name
print("Started %s \t Run durations" % (key))
if not self.bacteria.config[key].run_behaviour:
print("%s is non-motile, ignoring" % (key))
continue
output = os.path.join(self.graph_dir,
'%s_run_duration.png' % (key))
gp.c('set output "%s"' % (output))
title = 'Analysis of Run duration - %s'\
% (key)
gp.c('set title "%s"' % (title))
time_list = parallel(
delayed(Analysis.GetTimes)
(self.bacteria.run_log[key][bact],
self.bacteria.config[key])
for bact in self.bacteria.run_log[key].keys())
time_array = []
for i in range(len(time_list)):
time_array = np.append(time_array, time_list[i])
time_mean = np.mean(time_array)
time_std = np.std(time_array)
time_std_err = time_std / (np.sqrt(len(time_array)))
self.LDValues[key].avg_run_duration = time_mean
self.LDValues[key].run_dur_err = time_std_err
time_med = np.median(time_array)
results, bin_edges = np.histogram(time_array,
bins='auto',
density=True)
time_points = np.delete(
bin_edges + np.mean(np.diff(bin_edges)) / 2, -1)
graph_out = np.vstack((time_points, results))
title = "%s, Mean = %6.2f, std. dev. = %6.2f, Median = %6.2f"\
% (key_title, time_mean, time_std, time_med)
dat_name = os.path.join(self.plot_dir,
'%s_runduration_hist.dat' % (key))
gp.s(graph_out, dat_name)
plot_string = 'plot "%s" u 1:2 with points title "%s"'\
% (dat_name, title)
gp.c(plot_string)
amalg_dat_name.append(dat_name)
amalg_titles.append('title "%s"' % (title))
print("Completed %s \t Run duration" % (key))
if amalg_dat_name:
amalg_formatting = 'u 1:2 with points'
amalg_plot_string = plotStringMultiFile(
len(amalg_dat_name), amalg_dat_name, amalg_formatting,
amalg_titles)
output = os.path.join(self.graph_dir, 'run_duration_amalg.png')
gp.c('set output "%s"' % (output))
g_title = 'Analysis of Run duration'
gp.c('set title "%s"' % (g_title))
gp.c('set xrange [*:*]')
gp.c(amalg_plot_string)
# %%
gp.c('reset')
gp.c('set key top left')
gp.c("set terminal pngcairo enhanced" +
" size 1600,1200 font 'ariel, 14'")
gp.c('set logscale y')
gp.c('set ylabel "Probability Density"')
gp.c('set xlabel "Tumble Duration (s)"')
gp.c('set xrange [*:*]')
amalg_dat_name = []
amalg_titles = []
for key in self.bacteria.bacterium.keys():
key_title = self.bacteria.config[key].name
print("Started %s \t Tumble durations" % (key))
if not self.bacteria.config[key].run_behaviour:
print("%s is non-motile, ignoring" % (key))
continue
if self.bacteria.config[key].archaea_mode:
print("%s is an archaea, ignoring" % (key))
continue
output = os.path.join(self.graph_dir,
'%s_tumble_duration.png' % (key))
gp.c('set output "%s"' % (output))
title = 'Analysis of Tumble duration - %s'\
% (key_title)
gp.c('set title "%s"' % (title))
time_list = parallel(
delayed(Analysis.GetTimes)
(self.bacteria.tumble_log[key][bact],
self.bacteria.config[key])
for bact in self.bacteria.run_log[key].keys())
time_array = []
for i in range(len(time_list)):
time_array = np.append(time_array, time_list[i])
time_mean = np.mean(time_array)
self.LDValues[key].avg_tumble_duration = time_mean
time_std = np.std(time_array)
time_med = np.median(time_array)
results, bin_edges = np.histogram(time_array,
bins='auto',
density=True)
time_points = np.delete(
bin_edges + np.mean(np.diff(bin_edges)) / 2, -1)
graph_out = np.vstack((time_points, results))
title = "%s, Mean = %6.2f, std. dev. = %6.2f, Median = %6.2f"\
% (key_title, time_mean, time_std, time_med)
dat_name = os.path.join(self.plot_dir,
'%s_tumbleduration_hist.dat' % (key))
gp.s(graph_out, dat_name)
plot_string = 'plot "%s" u 1:2 with points title "%s"'\
% (dat_name, title)
gp.c(plot_string)
amalg_dat_name.append(dat_name)
amalg_titles.append('title "%s"' % (title))
print("Completed %s \t Tumble duration" % (key))
amalg_formatting = 'u 1:2 with points'
if amalg_dat_name:
amalg_plot_string = plotStringMultiFile(
len(amalg_dat_name), amalg_dat_name, amalg_formatting,
amalg_titles)
output = os.path.join(self.graph_dir,
'tumble_duration_amalg.png')
gp.c('set output "%s"' % (output))
g_title = 'Analysis of Tumble duration'
gp.c('set title "%s"' % (g_title))
gp.c('set xrange [*:*]')
gp.c(amalg_plot_string)
LD_file = os.path.join(self.plot_dir, "LD_values.txt")
with open(LD_file, "w") as LD_f:
for key in self.bacteria.bacterium.keys():
if self.bacteria.config[key].run_behaviour:
self.LDValues[key].LDCalc()
write_string = "%s \t Name: %s \tLD Value: %e\n"\
% (key, self.bacteria.config[key].name,
self.LDValues[key].LD_Diff)
LD_f.write(write_string)
import numpy as np
import Analysis
import Import
import BP
print("正在训练神经网络")
# 训练数据输入
TrainSet_x = Import.read_case('DataFile/StatusAndResult.xls', 'Sheet1_x')
TrainSet_y = Import.read_case('DataFile/StatusAndResult.xls', 'Sheet2_y')
# 训练神经网络
nt = BP.training(TrainSet_x, TrainSet_y)
# 训练结果保存
Import.write_syn('DataFile/Syn.xls', nt.weights)
# 测试数据输入
Test_input = np.array(Import.read_case('DataFile/Test.xls', 'Sheet1_x'))
Test_output = Import.read_case('DataFile/Test.xls', 'Sheet2_y')
# 测试数据输出
result = BP.forward(Test_input, nt)[nt.num_layers - 1]
# 测试结果整理
result_sort = Analysis.sort(result)
print("\n整理后的测试结果\n", result_sort)
# 正确率计算
rate = Analysis.rate(result, Test_output)
print("\n测试正确率为:\n", rate)
import Analysis
import matplotlib as mpl
from matplotlib import pyplot as plt
import numpy as np
import os
run14 = Analysis.Run(np.load('2014_Raw.npy'), np.load('2014_Prune_strong.npy'),
Analysis.bval2014, np.load('2014bf_new_prune_edit.npy'))
run11 = Analysis.Run(np.load('2011_Raw.npy'),
np.load('2011_Prune_strong_flat.npy'), Analysis.bval2011,
np.load('2011bf_new_prune_flat_edit.npy'))
i14 = run14.prune_data
i11 = run11.prune_data
print('Data loaded')
os.chdir('fitcheck2019')
print('Moved Dir')
print('2014 Data')
fig = plt.figure()
n = 0
for i in i14:
k = 0
b = Analysis.bval2014[n]
for j in i.fits:
i.plot()
j.plot(False)
plt.title('B:{},{}'.format(
def setup():
nexus = Nexus()
problem = Data()
nexus.optimization_problem = problem
# -------------------------------------------------------------------
# Inputs
# -------------------------------------------------------------------
# [ tag , initial, (lb,ub) , scaling , units ]
problem.inputs = np.array([
[ 'wing_area' , 95 , ( 90. , 130. ) , 100. , Units.meter**2],
[ 'cruise_altitude' , 11 , ( 9 , 14. ) , 10. , Units.km],
])
# -------------------------------------------------------------------
# Objective
# -------------------------------------------------------------------
# throw an error if the user isn't specific about wildcards
# [ tag, scaling, units ]
problem.objective = np.array([
[ 'fuel_burn', 10000, Units.kg ]
])
# -------------------------------------------------------------------
# Constraints
# -------------------------------------------------------------------
# [ tag, sense, edge, scaling, units ]
problem.constraints = np.array([
[ 'design_range_fuel_margin' , '>', 0., 1E-1, Units.less], #fuel margin defined here as fuel
])
# -------------------------------------------------------------------
# Aliases
# -------------------------------------------------------------------
# [ 'alias' , ['data.path1.name','data.path2.name'] ]
problem.aliases = [
[ 'wing_area' , ['vehicle_configurations.*.wings.main_wing.areas.reference',
'vehicle_configurations.*.reference_area' ]],
[ 'cruise_altitude' , 'missions.base.segments.climb_5.altitude_end' ],
[ 'fuel_burn' , 'summary.base_mission_fuelburn' ],
[ 'design_range_fuel_margin' , 'summary.max_zero_fuel_margin' ],
]
# -------------------------------------------------------------------
# Vehicles
# -------------------------------------------------------------------
# nexus.vehicle_configurations = Vehicles.setup()
vehicle = plane.getPlane()
nexus.vehicle_configurations = configurations.configSetup(vehicle)
# -------------------------------------------------------------------
# Analyses
# -------------------------------------------------------------------
nexus.analyses = Analysis.setup(nexus.vehicle_configurations)
# -------------------------------------------------------------------
# Missions
# -------------------------------------------------------------------
nexus.missions = Missions.setup(nexus.analyses)
# -------------------------------------------------------------------
# Procedure
# -------------------------------------------------------------------
nexus.procedure = Procedure.setup()
# -------------------------------------------------------------------
# Summary
# -------------------------------------------------------------------
nexus.summary = Data()
nexus.total_number_of_iterations = 0
return nexus
contractJ1sql = "'Joe Biden'"
contractJ2sql = "'Joe Biden'"
marketJ2sql = "'Who will win the 2020 Democratic presidential nomination?'"
marketJ1sql = "'Who will win the 2020 U.S. presidential election?'"
contractJ1 = 'Joe Biden'
contractJ2 = 'Joe Biden'
marketJ2 = 'Who will win the 2020 Democratic presidential nomination?'
marketJ1 = 'Who will win the 2020 U.S. presidential election?'
csvfilenow = 'BidenAnalysis.csv'
csvhist = 'biden.csv'
buyaggros = 0.3
sellaggros = 0.3
zztop = 1
s1name = 'Joe Biden for President 2020'
s2name = 'Joe Biden for Democratic Nomination 2020'
# In[ ]:
an.analysis_loop(csvhist, csvfilenow, marketJ1sql, marketJ2sql,contractJ1sql,
contractJ2sql, marketJ1, marketJ2, contractJ1, contractJ2,
buyaggros, sellaggros, zztop, s1name, s2name)
# In[4]:
import os
if os.environ.has_key('REPack'):
sys.path.append(os.environ['REPack'])
else:
sys.path.append(r'..\\')
import pprint
import logging
import json
import Analysis
logging.basicConfig(level=logging.DEBUG)
logger = logging.getLogger(__name__)
if __name__ == '__main__':
disasm = Analysis.Disasm()
save_filename = ''
if len(disasm.Args) > 0:
save_filename = disasm.Args[0]
else:
import UI
global form
title = 'Breakpoints-UI'
try:
form
form.OnClose(form)
form = UI.Form(title)
except:
import Analysis, sys, math, numpy as np, pylab
mapping = Analysis.loadmapping(sys.argv[1])
num_rows = 256
num_cols = 512
colors = np.zeros((256,512,3))
starty = np.random.rand(1,100)*150 + 50
startx = np.random.rand(1,100)*400 + 50
endx, endy = mapping(startx, starty)
base_shift_x = 256 # (endx - startx).mean()
base_shift_y = 512 # (endy - starty).mean()
for x in range(num_cols):
for y in range(num_rows):
newx, newy = mapping(x,y)
sat = 1
hue = (90 + 180 / math.pi * math.atan2(newy - base_shift_y, newx - base_shift_x)) % 360
val = min(.001 * math.sqrt((newy - base_shift_y)**2 + (newx - base_shift_x)**2), 1)
hueprm = hue / 60
C = val * sat
X = C * (1 - abs((hueprm %2) -1 ))
if hueprm < 1: r1, g1, b1 = C, X, 0
elif hueprm < 2: r1, g1, b1 = X, C, 0
elif hueprm < 3: r1, g1, b1 = X, C, 0
# def __init__(self,conf, game10,game20,modifier='nothing',verbose0=False):
ThisPebble = PB.Pebbles(ThisConf, 3, 3, 'nothing', False)
# play game
ThisPebble.play_game()
# compute rigid clusters
cidx, clusterall, clusterallBonds, clusteridx, BigCluster = ThisPebble.rigid_cluster(
)
########### Setting up the dynamical matrix and getting eigenmodes
# This itself does very little, just creates an empty Hessian class
# __init__(self,conf0):
ThisHessian = HS.Hessian(ThisConf)
########## Have a look at some analysis functions of the rigid clusters
#def __init__(self,conf0,pebbles0,hessian0,verbose=False):
ThisAnalysis = AN.Analysis(ThisConf, ThisPebble, ThisHessian, 0.01,
False)
# stress statistics
zav, nm, pres, fxbal, fybal, torbal, mobin, mohist, sxx, syy, sxy, syx = ThisAnalysis.getStressStat(
)
# cluster statistics
frac, fracmax, lenx, leny = ThisAnalysis.clusterStatistics()
#def plotStresses(self,plotCir,plotVel,plotCon,plotF,plotStress,**kwargs):
fig1 = ThisAnalysis.plotStresses(True, False, False, True, False)
#def plotPebbles(self,plotCir,plotPeb,plotPebCon,plotClus,plotOver,**kwargs):
#ThisAnalysis.plotPebbles(True,True,True,False,False)
fig2 = ThisAnalysis.plotPebbles(True, True, False, True, False)
######### continuing with the Hessian now
# constructing the matrix
# makeHessian(self,frictional,recomputeFnor,stabilise,verbose=False):
ThisHessian.makeHessian(True, False, 0, True)
import Analysis
import math
import json
import heapq
import numpy as np
top_k = 1000
#############################
word_list = []
avg = sum(Analysis.vectorizer.idf_) / Analysis.vectorizer.idf_.__len__()
scores = Analysis.get_scores()
print("Analysis done....")
writer1 = open('scores.txt', 'w', encoding="utf8")
writer = open('words.txt', 'w', encoding="utf8")
def cosine_similarity(v1, v2):
"compute cosine similarity of v1 to v2: (v1 dot v2)/{||v1||*||v2||)"
sumxx, sumxy, sumyy = 0, 0, 0
for i in range(len(v1)):
x = v1[i]
y = v2[i]
sumxx += x * x
sumyy += y * y
sumxy += x * y
help = "Maximum allowed inter-color colocalization distance",
default = 100, type = "float")
parser.add_option('-q', '--quiet', dest = "quiet",
action = "store_true", default = False)
parser.add_option('-d', '--quiver-distance', dest="quiv_dist",
help = "Minimum distance between points on the quiver map",
default = 10)
opts, args = parser.parse_args()
if not opts.map:
if opts.x1 and opts.x2 and opts.y1 and opts.y2:
opts.xshift = opts.x2 - opts.x1
opts.yshift = opts.y2 - opts.y1
else:
mapping = Analysis.loadmapping(opts.map)
if opts.map2:
mapping2 = Analysis.loadmapping(opts.map2)
if len(args) == 1 and '*' in args[0]:
args = glob(args[0])
print "Found this many files", len(args)
xrs = []
xls = []
yrs = []
yls = []
varxrs = []
varxls = []
for fname in args:
oscars_winloss[y] = Player3.get_wins() - Player3.get_losses()
safe_winloss[y] = Player4.get_wins() - Player4.get_losses()
martingale_streak[y] = Player1.get_win_streak()
manhattan_streak[y] = Player2.get_win_streak()
oscars_streak[y] = Player3.get_win_streak()
safe_streak[y] = Player4.get_win_streak()
# reset all players to defaults
Player1.__init__(p1_pot, min_bet, p1_wins, p1_losses, win_streak, profit)
Player2.__init__(p2_pot, min_bet, p2_wins, p2_losses, win_streak, profit)
Player3.__init__(p3_pot, min_bet, p3_wins, p3_losses, win_streak, profit)
Player4.__init__(p4_pot, min_bet, p4_wins, p4_losses, win_streak, profit)
Dealer.__init__(d_pot, d_wins, d_losses)
# conduct analysis
# generate analysis objects
Martingale_Stats = Analysis.AnalyticRecords(martingale_pot, martingale_winloss,
martingale_streak)
Manhattan_Stats = Analysis.AnalyticRecords(manhattan_pot, manhattan_winloss,
manhattan_streak)
Oscar_Stats = Analysis.AnalyticRecords(oscars_pot, oscars_winloss,
oscars_streak)
Safe_Stats = Analysis.AnalyticRecords(safe_pot, safe_winloss, safe_streak)
# part A - generate descriptive statistics and build confidence intervals
z_val = calc_z_val(alpha)
# mean, variance, and 95% confidence intervals for each strategy
# martingale strategy
print("Martingale strategy pot: mean " + str(Martingale_Stats.get_pot_mean()) +
", var " + str(Martingale_Stats.get_pot_var()))
print("Martingale strategy win/loss: mean " +
str(Martingale_Stats.get_winloss_mean()) + ", var " +
str(Martingale_Stats.get_winloss_var()))
print("Martingale strategy streak: mean " +
print 'Processing trajectory...',
sampler.traj.process() # compute averages, etc.
print '...Done.'
print 'Writing results...',
sampler.traj.write_results(os.path.join(outdir,'traj_lambda%2.2f.npz'%lam))
print '...Done.'
sampler.traj.read_results(os.path.join(outdir,'traj_lambda%2.2f.npz'%lam))
print 'Pickling the sampler object ...',
outfilename = 'sampler_lambda%2.2f.pkl'%lam
print outfilename,
fout = open(os.path.join(outdir, outfilename), 'wb')
# Pickle dictionary using protocol 0.
cPickle.dump(sampler, fout)
fout.close()
print '...Done.'
#########################################
# Let's do analysis using MBAR and plot figures
############ MBAR and Figures ###########
# Specify necessary argument values
A = Analysis(100,dataFiles,outdir)
A.plot()
def setUp(self):
self.analysis = Analysis.Analysis()
class DisassemblerView(QAbstractScrollArea):
statusUpdated = Signal(QWidget, name="statusUpdated")
def __init__(self, data, filename, view, parent):
super(DisassemblerView, self).__init__(parent)
self.status = ""
self.view = view
self.data = data
for type in ExeFormats:
exe = type(data)
if exe.valid:
self.data = exe
self.view.exe = exe
break
# Create analysis and start it in another thread
self.analysis = Analysis(self.data)
self.analysis_thread = threading.Thread(None, self.analysis_thread_proc)
self.analysis_thread.daemon = True
self.analysis_thread.start()
# Start disassembly view at the entry point of the binary
if hasattr(self.data, "entry"):
self.function = self.data.entry()
else:
self.function = None
self.update_id = None
self.ready = False
self.desired_pos = None
self.highlight_token = None
self.cur_instr = None
self.scroll_mode = False
self.blocks = {}
self.show_il = False
self.simulation = None
# Create timer to automatically refresh view when it needs to be updated
self.updateTimer = QTimer()
self.updateTimer.setInterval(100)
self.updateTimer.setSingleShot(False)
self.updateTimer.timeout.connect(self.updateTimerEvent)
self.updateTimer.start()
self.initFont()
# Initialize scroll bars
self.width = 0
self.height = 0
self.setHorizontalScrollBarPolicy(Qt.ScrollBarAsNeeded)
self.setVerticalScrollBarPolicy(Qt.ScrollBarAsNeeded)
self.horizontalScrollBar().setSingleStep(self.charWidth)
self.verticalScrollBar().setSingleStep(self.charHeight)
areaSize = self.viewport().size()
self.adjustSize(areaSize.width(), areaSize.height())
# Setup navigation
self.view.register_navigate("disassembler", self, self.navigate)
self.view.register_navigate("make_proc", self, self.make_proc)
self.search_regex = None
self.last_search_type = FindDialog.SEARCH_HEX
def initFont(self):
# Get font and compute character sizes
self.font = getMonospaceFont()
self.baseline = int(QFontMetricsF(self.font).ascent())
self.charWidth = QFontMetricsF(self.font).width('X')
self.charHeight = int(QFontMetricsF(self.font).height()) + getExtraFontSpacing()
self.charOffset = getFontVerticalOffset()
def adjustSize(self, width, height):
# Recompute size information
self.renderWidth = self.width
self.renderHeight = self.height
self.renderXOfs = 0
self.renderYOfs = 0
if self.renderWidth < width:
self.renderXOfs = int((width - self.renderWidth) / 2)
self.renderWidth = width
if self.renderHeight < height:
self.renderYOfs = int((height - self.renderHeight) / 2)
self.renderHeight = height
# Update scroll bar information
self.horizontalScrollBar().setPageStep(width)
self.horizontalScrollBar().setRange(0, self.renderWidth - width)
self.verticalScrollBar().setPageStep(height)
self.verticalScrollBar().setRange(0, self.renderHeight - height)
def resizeEvent(self, event):
# Window was resized, adjust scroll bar
self.adjustSize(event.size().width(), event.size().height())
def get_cursor_pos(self):
if self.cur_instr is None:
return self.function
return self.cur_instr
def set_cursor_pos(self, addr):
if not self.view.navigate("disassembler", addr):
self.view_in_hex_editor(addr)
def get_selection_range(self):
return (self.get_cursor_pos(), self.get_cursor_pos())
def set_selection_range(self, begin, end):
self.set_cursor_pos(begin)
def write(self, data):
pos = self.get_cursor_pos()
if pos is None:
return False
return self.data.write(pos, data) == len(data)
def copy_address(self):
clipboard = QApplication.clipboard()
clipboard.clear()
mime = QMimeData()
mime.setText("0x%x" % self.get_cursor_pos())
clipboard.setMimeData(mime)
def analysis_thread_proc(self):
self.analysis.analyze()
def closeRequest(self):
# Stop analysis when closing tab
self.analysis.stop()
return True
def paintEvent(self, event):
# Initialize painter
p = QPainter(self.viewport())
p.setFont(self.font)
xofs = self.horizontalScrollBar().value()
yofs = self.verticalScrollBar().value()
if not self.ready:
# Analysis for the current function is not yet complete, paint loading screen
gradient = QLinearGradient(QPointF(0, 0), QPointF(self.viewport().size().width(), self.viewport().size().height()))
gradient.setColorAt(0, QColor(232, 232, 232))
gradient.setColorAt(1, QColor(192, 192, 192))
p.setPen(QColor(0, 0, 0, 0))
p.setBrush(QBrush(gradient))
p.drawRect(0, 0, self.viewport().size().width(), self.viewport().size().height())
if self.function is None:
text = "No function selected"
else:
text = "Loading..."
p.setPen(Qt.black)
p.drawText((self.viewport().size().width() / 2) - ((len(text) * self.charWidth) / 2),
(self.viewport().size().height() / 2) + self.charOffset + self.baseline - (self.charHeight / 2), text)
return
# Render background
gradient = QLinearGradient(QPointF(-xofs, -yofs), QPointF(self.renderWidth - xofs, self.renderHeight - yofs))
gradient.setColorAt(0, QColor(232, 232, 232))
gradient.setColorAt(1, QColor(192, 192, 192))
p.setPen(QColor(0, 0, 0, 0))
p.setBrush(QBrush(gradient))
p.drawRect(0, 0, self.viewport().size().width(), self.viewport().size().height())
p.translate(self.renderXOfs - xofs, self.renderYOfs - yofs)
# Render each node
for block in self.blocks.values():
# Render shadow
p.setPen(QColor(0, 0, 0, 0))
p.setBrush(QColor(0, 0, 0, 128))
p.drawRect(block.x + self.charWidth + 4, block.y + self.charWidth + 4,
block.width - (4 + 2 * self.charWidth), block.height - (4 + 2 * self.charWidth))
# Render node background
gradient = QLinearGradient(QPointF(0, block.y + self.charWidth),
QPointF(0, block.y + block.height - self.charWidth))
gradient.setColorAt(0, QColor(255, 255, 252))
gradient.setColorAt(1, QColor(255, 255, 232))
p.setPen(Qt.black)
p.setBrush(QBrush(gradient))
p.drawRect(block.x + self.charWidth, block.y + self.charWidth,
block.width - (4 + 2 * self.charWidth), block.height - (4 + 2 * self.charWidth))
if self.cur_instr != None:
y = block.y + (2 * self.charWidth) + (len(block.block.header_text.lines) * self.charHeight)
for instr in block.block.instrs:
if instr.addr == self.cur_instr:
p.setPen(QColor(0, 0, 0, 0))
p.setBrush(QColor(255, 255, 128, 128))
p.drawRect(block.x + self.charWidth + 3, y, block.width - (10 + 2 * self.charWidth),
len(instr.text.lines) * self.charHeight)
y += len(instr.text.lines) * self.charHeight
if self.highlight_token:
# Render highlighted tokens
x = block.x + (2 * self.charWidth)
y = block.y + (2 * self.charWidth)
for line in block.block.header_text.tokens:
for token in line:
if token[2:] == self.highlight_token:
p.setPen(QColor(0, 0, 0, 0))
p.setBrush(QColor(192, 0, 0, 64))
p.drawRect(x + token[0] * self.charWidth, y,
token[1] * self.charWidth, self.charHeight)
y += self.charHeight
for instr in block.block.instrs:
for line in instr.text.tokens:
for token in line:
if token[2:] == self.highlight_token:
p.setPen(QColor(0, 0, 0, 0))
p.setBrush(QColor(192, 0, 0, 64))
p.drawRect(x + token[0] * self.charWidth, y,
token[1] * self.charWidth, self.charHeight)
y += self.charHeight
# Render node text
x = block.x + (2 * self.charWidth)
y = block.y + (2 * self.charWidth)
for line in block.block.header_text.lines:
partx = x
for part in line:
p.setPen(part[1])
p.drawText(partx, y + self.charOffset + self.baseline, part[0])
partx += len(part[0]) * self.charWidth
y += self.charHeight
for instr in block.block.instrs:
for line in instr.text.lines:
partx = x
for part in line:
p.setPen(part[1])
p.drawText(partx, y + self.charOffset + self.baseline, part[0])
partx += len(part[0]) * self.charWidth
y += self.charHeight
# Render edges
for edge in block.edges:
p.setPen(edge.color)
p.setBrush(edge.color)
p.drawPolyline(edge.polyline)
p.drawConvexPolygon(edge.arrow)
def isMouseEventInBlock(self, event):
# Convert coordinates to system used in blocks
xofs = self.horizontalScrollBar().value()
yofs = self.verticalScrollBar().value()
x = event.x() + xofs - self.renderXOfs
y = event.y() + yofs - self.renderYOfs
# Check each block for hits
for block in self.blocks.values():
# Compute coordinate relative to text area in block
blockx = x - (block.x + (2 * self.charWidth))
blocky = y - (block.y + (2 * self.charWidth))
# Check to see if click is within bounds of block
if (blockx < 0) or (blockx > (block.width - 4 * self.charWidth)):
continue
if (blocky < 0) or (blocky > (block.height - 4 * self.charWidth)):
continue
return True
return False
def getInstrForMouseEvent(self, event):
# Convert coordinates to system used in blocks
xofs = self.horizontalScrollBar().value()
yofs = self.verticalScrollBar().value()
x = event.x() + xofs - self.renderXOfs
y = event.y() + yofs - self.renderYOfs
# Check each block for hits
for block in self.blocks.values():
# Compute coordinate relative to text area in block
blockx = x - (block.x + (2 * self.charWidth))
blocky = y - (block.y + (2 * self.charWidth))
# Check to see if click is within bounds of block
if (blockx < 0) or (blockx > (block.width - 4 * self.charWidth)):
continue
if (blocky < 0) or (blocky > (block.height - 4 * self.charWidth)):
continue
# Compute row within text
row = int(blocky / self.charHeight)
# Determine instruction for this row
cur_row = len(block.block.header_text.lines)
if row < cur_row:
return block.block.entry
for instr in block.block.instrs:
if row < cur_row + len(instr.text.lines):
return instr.addr
cur_row += len(instr.text.lines)
return None
def getTokenForMouseEvent(self, event):
# Convert coordinates to system used in blocks
xofs = self.horizontalScrollBar().value()
yofs = self.verticalScrollBar().value()
x = event.x() + xofs - self.renderXOfs
y = event.y() + yofs - self.renderYOfs
# Check each block for hits
for block in self.blocks.values():
# Compute coordinate relative to text area in block
blockx = x - (block.x + (2 * self.charWidth))
blocky = y - (block.y + (2 * self.charWidth))
# Check to see if click is within bounds of block
if (blockx < 0) or (blockx > (block.width - 4 * self.charWidth)):
continue
if (blocky < 0) or (blocky > (block.height - 4 * self.charWidth)):
continue
# Compute row and column within text
col = int(blockx / self.charWidth)
row = int(blocky / self.charHeight)
# Check tokens to see if one was clicked
cur_row = 0
for line in block.block.header_text.tokens:
if cur_row == row:
for token in line:
if (col >= token[0]) and (col < (token[0] + token[1])):
# Clicked on a token
return token
cur_row += 1
for instr in block.block.instrs:
for line in instr.text.tokens:
if cur_row == row:
for token in line:
if (col >= token[0]) and (col < (token[0] + token[1])):
# Clicked on a token
return token
cur_row += 1
return None
def find_instr(self, addr):
for block in self.blocks.values():
for instr in block.block.instrs:
if instr.addr == addr:
return instr
return None
def nop_out(self, addr):
instr = self.find_instr(addr)
if instr != None:
self.view.begin_undo()
instr.patch_to_nop(self.data)
self.view.commit_undo()
def always_branch(self, addr):
instr = self.find_instr(addr)
if instr != None:
self.view.begin_undo()
instr.patch_to_always_branch(self.data)
self.view.commit_undo()
def invert_branch(self, addr):
instr = self.find_instr(addr)
if instr != None:
self.view.begin_undo()
instr.patch_to_invert_branch(self.data)
self.view.commit_undo()
def skip_and_return_zero(self, addr):
instr = self.find_instr(addr)
if instr != None:
self.view.begin_undo()
instr.patch_to_zero_return(self.data)
self.view.commit_undo()
def skip_and_return_value(self, addr):
instr = self.find_instr(addr)
if instr != None:
value, ok = QInputDialog.getText(self, "Skip and Return Value", "Return value:", QLineEdit.Normal)
if ok:
try:
value = int(value, 0)
except:
QMessageBox.critical(self, "Error", "Expected numerical address")
return
self.view.begin_undo()
instr.patch_to_fixed_return_value(self.data, value)
self.view.commit_undo()
def view_in_hex_editor(self, addr):
if not self.view.navigate("exe", addr):
self.view.navigate("hex", addr)
def show_address(self):
if "address" in self.analysis.options:
addr = False
else:
addr = True
self.analysis.set_address_view(addr)
def context_menu(self, addr):
popup = QMenu()
view_in_hex = popup.addAction("View in &hex editor")
view_in_hex.triggered.connect(lambda : self.view_in_hex_editor(addr))
view_in_hex.setShortcut(QKeySequence(Qt.Key_H))
popup.addAction("Copy address", self.copy_address)
enter_name_action = popup.addAction("Re&name symbol", self.enter_name)
enter_name_action.setShortcut(QKeySequence(Qt.Key_N))
undefine_name_action = popup.addAction("&Undefine symbol", self.undefine_name)
undefine_name_action.setShortcut(QKeySequence(Qt.Key_U))
show_address_action = popup.addAction("Show &address", self.show_address)
show_address_action.setCheckable(True)
show_address_action.setChecked("address" in self.analysis.options)
popup.addSeparator()
patch = popup.addMenu("&Patch")
patch.addAction("Convert to NOP").triggered.connect(lambda : self.nop_out(addr))
instr = self.find_instr(addr)
if instr:
if instr.is_patch_branch_allowed():
patch.addAction("Never branch").triggered.connect(lambda : self.nop_out(addr))
patch.addAction("Always branch").triggered.connect(lambda : self.always_branch(addr))
patch.addAction("Invert branch").triggered.connect(lambda : self.invert_branch(addr))
if instr.is_patch_to_zero_return_allowed():
patch.addAction("Skip and return zero").triggered.connect(lambda : self.skip_and_return_zero(addr))
if instr.is_patch_to_fixed_return_value_allowed():
patch.addAction("Skip and return value...").triggered.connect(lambda : self.skip_and_return_value(addr))
popup.exec_(QCursor.pos())
def mousePressEvent(self, event):
if (event.button() != Qt.LeftButton) and (event.button() != Qt.RightButton):
return
if not self.isMouseEventInBlock(event):
# Click outside any block, enter scrolling mode
self.scroll_base_x = event.x()
self.scroll_base_y = event.y()
self.scroll_mode = True
self.viewport().grabMouse()
return
# Check for click on a token and highlight it
token = self.getTokenForMouseEvent(event)
if token:
self.highlight_token = token[2:]
else:
self.highlight_token = None
# Update current instruction
instr = self.getInstrForMouseEvent(event)
if instr != None:
self.cur_instr = instr
else:
self.cur_instr = None
self.viewport().update()
if (instr != None) and (event.button() == Qt.RightButton):
self.context_menu(instr)
def mouseMoveEvent(self, event):
if self.scroll_mode:
x_delta = self.scroll_base_x - event.x()
y_delta = self.scroll_base_y - event.y()
self.scroll_base_x = event.x()
self.scroll_base_y = event.y()
self.horizontalScrollBar().setValue(self.horizontalScrollBar().value() + x_delta)
self.verticalScrollBar().setValue(self.verticalScrollBar().value() + y_delta)
def mouseReleaseEvent(self, event):
if event.button() != Qt.LeftButton:
return
if self.scroll_mode:
self.scroll_mode = False
self.viewport().releaseMouse()
def mouseDoubleClickEvent(self, event):
token = self.getTokenForMouseEvent(event)
if token and (token[2] == "ptr"):
self.analysis.lock.acquire()
if not self.analysis.functions.has_key(token[3]):
# Not a function or not analyzed, go to address in hex editor
addr = token[3]
self.analysis.lock.release()
self.view_in_hex_editor(addr)
else:
self.view.add_history_entry()
self.function = token[3]
self.ready = False
self.desired_pos = None
self.cur_instr = None
self.highlight_token = None
self.viewport().update()
self.analysis.lock.release()
def go_to_address(self):
addr_str, ok = QInputDialog.getText(self, "Go To Address", "Address:", QLineEdit.Normal)
if ok:
try:
addr = int(addr_str, 16)
if (addr < self.data.start()) or (addr > self.data.end()):
if hasattr(self.data, "symbols_by_name") and (addr_str in self.data.symbols_by_name):
addr = self.data.symbols_by_name[addr_str]
else:
QMessageBox.critical(self, "Error", "Address out of range")
return
except:
if hasattr(self.data, "symbols_by_name") and (addr_str in self.data.symbols_by_name):
addr = self.data.symbols_by_name[addr_str]
elif (addr_str[0] == '@') and hasattr(self.data, "symbols_by_name") and (addr_str[1:] in self.data.symbols_by_name):
addr = self.data.symbols_by_name[addr_str[1:]]
else:
QMessageBox.critical(self, "Error", "Invalid address or symbol")
return
# Try navigating within disassembly, if it isn't within a function then
# navigate to the hex editor
if not self.view.navigate("disassembler", addr):
self.view_in_hex_editor(addr)
def enter_name(self):
# A symbol must be selected
if (self.highlight_token == None) or (self.highlight_token[0] != "ptr"):
QMessageBox.critical(self, "Error", "No symbol selected.")
return
addr = self.highlight_token[1]
name = self.highlight_token[2]
# Ask for new name
new_name, ok = QInputDialog.getText(self, "Rename Symbol", "Symbol name:", QLineEdit.Normal, name)
if ok:
self.analysis.create_symbol(addr, new_name)
def undefine_name(self):
# A symbol must be selected
if (self.highlight_token == None) or (self.highlight_token[0] != "ptr"):
QMessageBox.critical(self, "Error", "No symbol selected.")
return
addr = self.highlight_token[1]
name = self.highlight_token[2]
# Ask for new name
self.analysis.undefine_symbol(addr, name)
def navigate_for_find(self, addr):
func, instr = self.analysis.find_instr(addr, True)
if func != None:
self.navigate(addr)
else:
self.make_proc(addr)
self.cur_instr = addr
self.desired_pos = None
def perform_find(self, dlg):
self.search_regex = dlg.search_regex()
if self.cur_instr != None:
self.search_start = self.cur_instr
else:
if self.function is None:
return
self.search_start = self.function
found_loc = self.data.find(self.search_regex, self.search_start)
if found_loc != -1:
self.view.add_history_entry()
self.navigate_for_find(found_loc)
self.search_pos = found_loc + 1
return
found_loc = self.data.find(self.search_regex, self.data.start())
if (found_loc != -1) and (found_loc < self.search_start):
self.view.add_history_entry()
self.navigate_for_find(found_loc)
self.search_pos = found_loc + 1
return
QMessageBox.information(self, "Not Found", "Search string not found.")
def find(self):
dlg = FindDialog(self.last_search_type, self)
if dlg.exec_() == QDialog.Accepted:
self.last_search_type = dlg.search_type()
self.perform_find(dlg)
def find_next(self):
if self.search_regex == None:
QMessageBox.critical(self, "Error", "No active search")
return
found_loc = self.data.find(self.search_regex, self.search_pos)
if self.search_pos >= self.search_start:
if found_loc != -1:
self.view.add_history_entry()
self.navigate_for_find(found_loc)
self.search_pos = found_loc + 1
return
self.search_pos = 0
else:
if (found_loc != -1) and (found_loc < self.search_start):
self.view.add_history_entry()
self.navigate_for_find(found_loc)
self.search_pos = found_loc + 1
return
QMessageBox.information(self, "End of Search", "No additional matches found.")
self.search_pos = self.search_start
return
found_loc = self.data.find(self.search_regex, self.search_pos)
if found_loc < self.search_start:
self.view.add_history_entry()
self.navigate_for_find(found_loc)
self.search_pos = found_loc + 1
return
QMessageBox.information(self, "End of Search", "No additional matches found.")
self.search_pos = self.search_start
def keyPressEvent(self, event):
if event.key() == Qt.Key_H:
if self.cur_instr != None:
self.view_in_hex_editor(self.cur_instr)
else:
if self.function is not None:
self.view_in_hex_editor(self.function)
elif event.key() == Qt.Key_G:
self.go_to_address()
elif event.key() == Qt.Key_N:
self.enter_name()
elif event.key() == Qt.Key_U:
self.undefine_name()
elif event.key() == Qt.Key_Slash:
dlg = FindDialog(FindDialog.SEARCH_REGEX, self)
if dlg.exec_() == QDialog.Accepted:
self.perform_find(dlg)
else:
super(DisassemblerView, self).keyPressEvent(event)
def prepareGraphNode(self, block):
# Compute size of node in pixels
width = 0
height = 0
for line in block.block.header_text.lines:
chars = 0
for part in line:
chars += len(part[0])
if chars > width:
width = chars
height += 1
for instr in block.block.instrs:
for line in instr.text.lines:
chars = 0
for part in line:
chars += len(part[0])
if chars > width:
width = chars
height += 1
block.width = (width + 4) * self.charWidth + 4
block.height = (height * self.charHeight) + (4 * self.charWidth) + 4
def adjustGraphLayout(self, block, col, row):
block.col += col
block.row += row
for edge in block.new_exits:
self.adjustGraphLayout(self.blocks[edge], col, row)
def computeGraphLayout(self, block):
# Compute child node layouts and arrange them horizontally
col = 0
row_count = 1
for edge in block.new_exits:
self.computeGraphLayout(self.blocks[edge])
self.adjustGraphLayout(self.blocks[edge], col, 1)
col += self.blocks[edge].col_count
if (self.blocks[edge].row_count + 1) > row_count:
row_count = self.blocks[edge].row_count + 1
block.row = 0
if col >= 2:
# Place this node centered over the child nodes
block.col = (col - 2) / 2
block.col_count = col
else:
# No child nodes, set single node's width (nodes are 2 columns wide to allow
# centering over a branch)
block.col = 0
block.col_count = 2
block.row_count = row_count
def isEdgeMarked(self, edges, row, col, index):
if index >= len(edges[row][col]):
return False
return edges[row][col][index]
def markEdge(self, edges, row, col, index):
while len(edges[row][col]) <= index:
edges[row][col] += [False]
edges[row][col][index] = True
def findHorizEdgeIndex(self, edges, row, min_col, max_col):
# Find a valid index
i = 0
while True:
valid = True
for col in range(min_col, max_col + 1):
if self.isEdgeMarked(edges, row, col, i):
valid = False
break
if valid:
break
i += 1
# Mark chosen index as used
for col in range(min_col, max_col + 1):
self.markEdge(edges, row, col, i)
return i
def findVertEdgeIndex(self, edges, col, min_row, max_row):
# Find a valid index
i = 0
while True:
valid = True
for row in range(min_row, max_row + 1):
if self.isEdgeMarked(edges, row, col, i):
valid = False
break
if valid:
break
i += 1
# Mark chosen index as used
for row in range(min_row, max_row + 1):
self.markEdge(edges, row, col, i)
return i
def routeEdge(self, horiz_edges, vert_edges, edge_valid, start, end, color):
edge = DisassemblerEdge(color, end)
# Find edge index for initial outgoing line
i = 0
while True:
if not self.isEdgeMarked(vert_edges, start.row + 1, start.col + 1, i):
break
i += 1
self.markEdge(vert_edges, start.row + 1, start.col + 1, i)
edge.addPoint(start.row + 1, start.col + 1)
edge.start_index = i
horiz = False
# Find valid column for moving vertically to the target node
if end.row < (start.row + 1):
min_row = end.row
max_row = start.row + 1
else:
min_row = start.row + 1
max_row = end.row
col = start.col + 1
if min_row != max_row:
ofs = 0
while True:
col = start.col + 1 - ofs
if col >= 0:
valid = True
for row in range(min_row, max_row + 1):
if not edge_valid[row][col]:
valid = False
break
if valid:
break
col = start.col + 1 + ofs
if col < len(edge_valid[min_row]):
valid = True
for row in range(min_row, max_row + 1):
if not edge_valid[row][col]:
valid = False
break
if valid:
break
ofs += 1
if col != (start.col + 1):
# Not in same column, need to generate a line for moving to the correct column
if col < (start.col + 1):
min_col = col
max_col = start.col + 1
else:
min_col = start.col + 1
max_col = col
index = self.findHorizEdgeIndex(horiz_edges, start.row + 1, min_col, max_col)
edge.addPoint(start.row + 1, col, index)
horiz = True
if end.row != (start.row + 1):
# Not in same row, need to generate a line for moving to the correct row
index = self.findVertEdgeIndex(vert_edges, col, min_row, max_row)
edge.addPoint(end.row, col, index)
horiz = False
if col != (end.col + 1):
# Not in ending column, need to generate a line for moving to the correct column
if col < (end.col + 1):
min_col = col
max_col = end.col + 1
else:
min_col = end.col + 1
max_col = col
index = self.findHorizEdgeIndex(horiz_edges, end.row, min_col, max_col)
edge.addPoint(end.row, end.col + 1, index)
horiz = True
# If last line was horizontal, choose the ending edge index for the incoming edge
if horiz:
index = self.findVertEdgeIndex(vert_edges, end.col + 1, end.row, end.row)
edge.points[len(edge.points) - 1][2] = index
return edge
def renderFunction(self, func):
# Create render nodes
self.blocks = {}
for block in func.blocks.values():
self.blocks[block.entry] = DisassemblerBlock(block)
self.prepareGraphNode(self.blocks[block.entry])
# Populate incoming lists
for block in self.blocks.values():
for edge in block.block.exits:
self.blocks[edge].incoming += [block.block.entry]
# Construct acyclic graph where each node is used as an edge exactly once
block = func.blocks[func.entry]
visited = [func.entry]
queue = [self.blocks[func.entry]]
changed = True
while changed:
changed = False
# First pick nodes that have single entry points
while len(queue) > 0:
block = queue.pop()
for edge in block.block.exits:
if edge in visited:
continue
# If node has no more unseen incoming edges, add it to the graph layout now
if len(self.blocks[edge].incoming) == 1:
self.blocks[edge].incoming.remove(block.block.entry)
block.new_exits += [edge]
queue += [self.blocks[edge]]
visited += [edge]
changed = True
# No more nodes satisfy constraints, pick a node to continue constructing the graph
best = None
for block in self.blocks.values():
if not block.block.entry in visited:
continue
for edge in block.block.exits:
if edge in visited:
continue
if (best == None) or (len(self.blocks[edge].incoming) < best_edges) or ((len(self.blocks[edge].incoming) == best_edges) and (edge < best)):
best = edge
best_edges = len(self.blocks[edge].incoming)
best_parent = block
if best != None:
self.blocks[best].incoming.remove(best_parent.block.entry)
best_parent.new_exits += [best]
visited += [best]
changed = True
# Compute graph layout from bottom up
self.computeGraphLayout(self.blocks[func.entry])
# Prepare edge routing
horiz_edges = [None] * (self.blocks[func.entry].row_count + 1)
vert_edges = [None] * (self.blocks[func.entry].row_count + 1)
edge_valid = [None] * (self.blocks[func.entry].row_count + 1)
for row in range(0, self.blocks[func.entry].row_count + 1):
horiz_edges[row] = [None] * (self.blocks[func.entry].col_count + 1)
vert_edges[row] = [None] * (self.blocks[func.entry].col_count + 1)
edge_valid[row] = [True] * (self.blocks[func.entry].col_count + 1)
for col in range(0, self.blocks[func.entry].col_count + 1):
horiz_edges[row][col] = []
vert_edges[row][col] = []
for block in self.blocks.values():
edge_valid[block.row][block.col + 1] = False
# Perform edge routing
for block in self.blocks.values():
start = block
for edge in block.block.exits:
end = self.blocks[edge]
color = Qt.black
if edge == block.block.true_path:
color = QColor(0, 144, 0)
elif edge == block.block.false_path:
color = QColor(144, 0, 0)
start.edges += [self.routeEdge(horiz_edges, vert_edges, edge_valid, start, end, color)]
# Compute edge counts for each row and column
col_edge_count = [0] * (self.blocks[func.entry].col_count + 1)
row_edge_count = [0] * (self.blocks[func.entry].row_count + 1)
for row in range(0, self.blocks[func.entry].row_count + 1):
for col in range(0, self.blocks[func.entry].col_count + 1):
if len(horiz_edges[row][col]) > row_edge_count[row]:
row_edge_count[row] = len(horiz_edges[row][col])
if len(vert_edges[row][col]) > col_edge_count[col]:
col_edge_count[col] = len(vert_edges[row][col])
# Compute row and column sizes
col_width = [0] * (self.blocks[func.entry].col_count + 1)
row_height = [0] * (self.blocks[func.entry].row_count + 1)
for block in self.blocks.values():
if (int(block.width / 2)) > col_width[block.col]:
col_width[block.col] = int(block.width / 2)
if (int(block.width / 2)) > col_width[block.col + 1]:
col_width[block.col + 1] = int(block.width / 2)
if int(block.height) > row_height[block.row]:
row_height[block.row] = int(block.height)
# Compute row and column positions
col_x = [0] * self.blocks[func.entry].col_count
row_y = [0] * self.blocks[func.entry].row_count
self.col_edge_x = [0] * (self.blocks[func.entry].col_count + 1)
self.row_edge_y = [0] * (self.blocks[func.entry].row_count + 1)
x = 16
for i in range(0, self.blocks[func.entry].col_count):
self.col_edge_x[i] = x
x += 8 * col_edge_count[i]
col_x[i] = x
x += col_width[i]
y = 16
for i in range(0, self.blocks[func.entry].row_count):
self.row_edge_y[i] = y
y += 8 * row_edge_count[i]
row_y[i] = y
y += row_height[i]
self.col_edge_x[self.blocks[func.entry].col_count] = x
self.row_edge_y[self.blocks[func.entry].row_count] = y
self.width = x + 16 + (8 * col_edge_count[self.blocks[func.entry].col_count])
self.height = y + 16 + (8 * row_edge_count[self.blocks[func.entry].row_count])
# Compute node positions
for block in self.blocks.values():
block.x = int((col_x[block.col] + col_width[block.col] + 4 * col_edge_count[block.col + 1]) - (block.width / 2))
if (block.x + block.width) > (col_x[block.col] + col_width[block.col] + col_width[block.col + 1] + 8 * col_edge_count[block.col + 1]):
block.x = int((col_x[block.col] + col_width[block.col] + col_width[block.col + 1] + 8 * col_edge_count[block.col + 1]) - block.width)
block.y = row_y[block.row]
# Precompute coordinates for edges
for block in self.blocks.values():
for edge in block.edges:
start = edge.points[0]
start_row = start[0]
start_col = start[1]
last_index = edge.start_index
last_pt = QPoint(self.col_edge_x[start_col] + (8 * last_index) + 4,
block.y + block.height + 4 - (2 * self.charWidth))
pts = [last_pt]
for i in range(0, len(edge.points)):
end = edge.points[i]
end_row = end[0]
end_col = end[1]
last_index = end[2]
if start_col == end_col:
new_pt = QPoint(last_pt.x(), self.row_edge_y[end_row] + (8 * last_index) + 4)
else:
new_pt = QPoint(self.col_edge_x[end_col] + (8 * last_index) + 4, last_pt.y())
pts += [new_pt]
last_pt = new_pt
start_col = end_col
new_pt = QPoint(last_pt.x(), edge.dest.y + self.charWidth - 1)
pts += [new_pt]
edge.polyline = pts
pts = [QPoint(new_pt.x() - 3, new_pt.y() - 6), QPoint(new_pt.x() + 3, new_pt.y() - 6), new_pt]
edge.arrow = pts
# Adjust scroll bars for new size
areaSize = self.viewport().size()
self.adjustSize(areaSize.width(), areaSize.height())
if self.desired_pos:
# There was a position saved, navigate to it
self.horizontalScrollBar().setValue(self.desired_pos[0])
self.verticalScrollBar().setValue(self.desired_pos[1])
elif self.cur_instr != None:
self.show_cur_instr()
else:
# Ensure start node is visible
start_x = self.blocks[func.entry].x + self.renderXOfs + int(self.blocks[func.entry].width / 2)
self.horizontalScrollBar().setValue(start_x - int(areaSize.width() / 2))
self.verticalScrollBar().setValue(0)
self.update_id = func.update_id
self.ready = True
self.viewport().update(0, 0, areaSize.width(), areaSize.height())
def updateTimerEvent(self):
status = self.analysis.status
if status != self.status:
self.status = status
self.statusUpdated.emit(self)
if self.function is None:
return
if self.ready:
# Check for updated code
self.analysis.lock.acquire()
if self.update_id != self.analysis.functions[self.function].update_id:
self.renderFunction(self.analysis.functions[self.function])
self.analysis.lock.release()
return
# View not up to date, check to see if active function is ready
self.analysis.lock.acquire()
if self.analysis.functions.has_key(self.function):
if self.analysis.functions[self.function].ready:
# Active function now ready, generate graph
self.renderFunction(self.analysis.functions[self.function])
self.analysis.lock.release()
def show_cur_instr(self):
for block in self.blocks.values():
row = len(block.block.header_text.lines)
for instr in block.block.instrs:
if self.cur_instr == instr.addr:
x = block.x + int(block.width / 2)
y = block.y + (2 * self.charWidth) + int((row + 0.5) * self.charHeight)
self.horizontalScrollBar().setValue(x + self.renderXOfs -
int(self.horizontalScrollBar().pageStep() / 2))
self.verticalScrollBar().setValue(y + self.renderYOfs -
int(self.verticalScrollBar().pageStep() / 2))
return
row += len(instr.text.lines)
def navigate(self, addr):
# Check to see if address is within current function
for block in self.blocks.values():
row = len(block.block.header_text.lines)
for instr in block.block.instrs:
if (addr >= instr.addr) and (addr < (instr.addr + len(instr.opcode))):
self.cur_instr = instr.addr
self.show_cur_instr()
self.viewport().update()
return True
row += len(instr.text.lines)
# Check other functions for this address
func, instr = self.analysis.find_instr(addr)
if func != None:
self.function = func
self.cur_instr = instr
self.highlight_token = None
self.ready = False
self.desired_pos = None
self.viewport().update()
return True
return False
def make_proc(self, addr):
# Create a procedure at the requested address if one does not already exist
if self.data.architecture() is None:
# Architecture not defined yet, ask the user and set it now
arch_dlg = ArchitectureDialog(self)
if arch_dlg.exec_() == QDialog.Rejected:
return False
self.data.default_arch = arch_dlg.result
self.analysis.lock.acquire()
if addr not in self.analysis.functions:
self.analysis.queue.append(addr)
self.analysis.lock.release()
self.function = addr
self.cur_instr = None
self.highlight_token = None
self.ready = False
self.desired_pos = None
self.viewport().update()
return True
def navigate_to_history_entry(self, entry):
self.function = entry.function
self.ready = False
self.desired_pos = [entry.scroll_x, entry.scroll_y]
self.cur_instr = entry.cur_instr
self.highlight_token = entry.highlight_token
self.viewport().update()
def get_history_entry(self):
return DisassemblerHistoryEntry(self)
def fontChanged(self):
self.initFont()
if self.ready:
# Rerender function to update layout
self.analysis.lock.acquire()
self.renderFunction(self.analysis.functions[self.function])
self.analysis.lock.release()
def getPriority(data, ext):
if Analysis.isPreferredForFile(data):
return 80
return 0
getPriority = staticmethod(getPriority)
def getViewName():
return "Disassembler"
getViewName = staticmethod(getViewName)
def getShortViewName():
return "Disassembler"
getShortViewName = staticmethod(getShortViewName)
def handlesNavigationType(name):
return (name == "disassembler") or (name == "make_proc")
handlesNavigationType = staticmethod(handlesNavigationType)
init_plot(i11[n])
i11[n].fits[j].plot_select('l', 'k', lw, -41.1981, -35.3895)
i11[n].fits[j].plot_select('l', 'k', lw, -35.3895, -31.6231, '--')
i11[n].fits[j].plot_select('r', 'r', lw, -41.1981, -35.3895, '--')
i11[n].fits[j].plot_select('r', 'r', lw, -35.3895, -31.6231)
set_axis(i11[n], -41.1981, -31.6231)
plt.xticks([-40, -35])
for k in se11[n][np.abs(se11[n][:, 2] - i11[n].fits[j].delta_phi) < 0.01]:
plt.axvline(k[0], color='k', lw=1)
i11 = Analysis.Run(np.load('2011_Raw.npy'),
np.load('2011_Prune_strong_flat.npy'), Analysis.bval2011,
np.load('2011bf_new_prune_flat_edit.npy')).prune_data
i14 = Analysis.Run(np.load('2014_Raw.npy'),
np.load('2014_Prune_strong.npy'), Analysis.bval2014,
np.load('2014bf_new_prune_edit.npy')).prune_data
soft11 = Analysis.Run(np.load('2011_Raw.npy'),
np.load('2011_Prune_new_flat.npy'), Analysis.bval2011,
np.load('2011bf_new_prune_flat_edit.npy')).prune_data
soft14 = Analysis.Run(np.load('2014_Raw.npy'),
np.load('2014_Prune_new.npy'), Analysis.bval2014,
np.load('2014bf_new_prune_edit.npy')).prune_data
se11 = np.load('2011se_flat_edit.npy')
se14 = np.load('2014se_edit.npy')
d140 = np.load('2014_0.npy')
d144 = np.load('2014_4.npy')
def askopenfile(self):
filename = tkinter.filedialog.askopenfilename(**self.file_opt)
Analysis.path = str(filename)
Analysis.main()
exit(0)
D = loadmat(filename, squeeze_me=True, struct_as_record=True)
xld = D["xl"]
yld = D["yl"]
xrd = D["xr"]
yrd = D["yr"]
varl = D["varxl"]
varr = D["varxr"]
del D
################################ Filter the Data ######################
# Get rid of anything where the width of the fit is too far
# outside of norms
varllo, varlhi = Analysis.middle_percent(abs(varl), 0.02)
varrlo, varrhi = Analysis.middle_percent(abs(varr), 0.02)
print "Variance tolerance left:", varllo, varlhi
print "Variance tolerance right:", varrlo, varrhi
var_sel = (varllo <= abs(varl)) * (abs(varl) <= varlhi) * (varrlo <= abs(varr)) * (abs(varr) <= varrhi)
stepper = rand(len(xld)) < frac
if opts.trim_edge: # Trim the outside border (where points are less reliable)
xlo, xhi = Analysis.middle_percent(xld, 0.02)
ylo, yhi = Analysis.middle_percent(yld, 0.02)
selA = (xlo < xld) * (xld < xhi) * (ylo < yld) * (yld < yhi)
# Get rid of anything where the match isn't actually that good.
xdiff = median(xld - xrd)
ydiff = median(yld - yrd)
midfield = fifaCleaner.convertHeightAndWeight(midfield)
defenders = fifaCleaner.ConvertMonetaryValue(defenders)
midfield = fifaCleaner.ConvertMonetaryValue(midfield)
defenders = defenders.fillna(defenders)
midfield = midfield.fillna(midfield)
missing_Defender_Data = util.Missing(defenders)
missing_midfielder_Data = util.Missing(midfield)
util.SideSide(missing_Defender_Data, missing_midfielder_Data)
print('Missing Data')
X_train, X_test, y_train, y_test = dc.Spliting(defenders, 'BP')
RF_Model = RandomForestClassifier(max_features='sqrt', max_leaf_nodes=5)
analysis.ApplyModel(X_train, y_train, RF_Model)
for position in arrayOfPositions:
players = fifaCleaner.getAllPlayersInPosition(position, fifaData)
playerDataFrame = pd.DataFrame(players)
print(playerDataFrame.info)
analysis.CorrelationMatrix('Overall', playerDataFrame)
index = arrayOfPositions.index(position)
if index == 0:
fileName = 'GoalKeepers'
elif index == 1:
fileName = 'Defenders'
elif index == 2:
fileName = 'Midfielders'
elif index == 3:
def main(self, option):
overall_bestfit = []
print("Please enter the maximum number of runs:")
maxruns = int(input())
print("Please enter desired number of generations: ")
gens = int(input())
print("Enter Population size")
self.population_size = int(input())
run_fitnesses = []
best_ind = []
for run in range(int(maxruns)):
# print("Run Number %i" % run)
generation = self.createPopulation(self.population_size,
self.len_of_ind)
print(generation)
run_fitnesses.clear()
for gen in range(int(gens)):
print(
"---------------------------------------------------------------"
)
#Genetic Algorithm part
x1, x2, x3 = self.extract_genes(generation)
fit_value = fit.calc_fitness_value(x1, x2, x3)
run_fitnesses.append(fit_value)
fitness_values = fit.generation_fitness_func(generation)
if option == "tournament":
generation = select.tournament_select(
generation, self.population_size)
if option == "linear":
generation = select.linear_rank_select(
generation, fitness_values, self.population_size)
if option == 'proportionate':
generation = select.proportionate_selection(
generation, fitness_values, self.population_size)
generation = Crossover.random_crossover(
generation, self.len_of_ind, self.population_size)
generation = self.mutation(generation, self.pm)
bor, vector = analyze.best_of_run(generation, run_fitnesses)
self.best_of_runs += [bor]
self.best_individuals += vector
print("Best of run is %i" % bor)
if self.exec_start == 0:
overall_bestfit = bor
self.exec_start = 1
if bor < overall_bestfit:
overall_bestfit = bor
print("Best fitness over all runs is %i" % overall_bestfit)
avg = analyze.mean(run_fitnesses)
print("Average of all runs is %f" % avg)
print("Best of run values:")
print(self.best_of_runs)
print("Standard deviation from {0} independent runs is {1}".format(
maxruns, np.std(self.best_of_runs)))
print("Best individuals: ")
for each in range(len(self.best_individuals)):
print(self.best_individuals[each])
class DisplayApp:
# init function
def __init__(self, width, height):
#motion ratio
self.kn = 0.95
self.Delta = np.matrix([0, 0, 0])
self.offset = np.matrix([20,20])
self.windowOffset = np.matrix([40,40])
self.check = .001
self.dxMin = 0.5
self.dxMax = 10
self.dxDefault = 3
#store figure
self.fig = None
# create a tk object, which is the root window
self.root = tk.Tk()
# width and height of the window
self.initDx = width
self.initDy = height
# set up the geometry for the window
self.root.geometry( "%dx%d+50+30" % (self.initDx, self.initDy) )
# set the title of the window
self.root.title("D.I.V.A")
# set the maximum size of the window for resizing
self.root.maxsize( 1024, 768 )
# bring the window to the front
self.root.lift()
# setup the menus
self.buildMenus()
# build the controls
self.buildControls()
#build point locator
self.buildLocation()
# build the objects on the Canvas
self.buildCanvas()
# set up the key bindings
self.setBindings()
# set up the application state
self.objects = []
self.data = tuple()
self.dataObject = None
#@add to build axes function
#create a view object
self.viewObj = View()
self.viewObj.setScreen(screen = (np.matrix([width, height])-self.windowOffset))
#print self.viewObj.getScreen()
#axes labels get initialized
self.xAxisLabel = StringVar()
self.xAxisLabel.set("x")
self.yAxisLabel = StringVar()
self.yAxisLabel.set("y")
self.zAxisLabel = StringVar()
self.zAxisLabel.set("z")
self.xlabel = None
self.ylabel = None
self.zlabel = None
#initialize axes
self.lines = []
#picture is displayed first
#self.buildAxes()
#self.buildLabels()
self.headerData = {}
#Initialize values for the plotting functions
self.size = None
self.color = None
self.normmatrix = None
self.Analysis = None
self.data_num = None
#self.image = ImageTk.PhotoImage(Image.open('Diva.jpg'))
self.image = Tkinter.PhotoImage(file = "diva.gif")
self.imageWidget = self.canvas.create_image(width/2,height/2,image=self.image)
#canvas.createLIne
# this method will test to see if the header given is a numeric header or not
# returns true if the header is a numeric dimension and false if it is not
def testHeader(self,header):
if(self.dataObject == None):
tkMessageBox.showwarning("Bad input","No data loaded")
return False
return self.dataObject.testHeader(header)
#this method will test to see if there is any data loaded.
# this returns true if there is data loaded, false otherwise
def dataTest(self):
if( self.dataObject == None):
return False
else:
return True
# this method will return a list of the headers
def getHeaders(self):
return self.dataObject.header_num()
# This method will build the bottom section of the display that will display the location in the graph of the point clicked on
# by the user when they use the control-mouse click option.
def buildLocation(self):
self.infoFrame = tk.Frame(self.root)
self.infoFrame.pack(side = tk.BOTTOM, padx=2,pady=2,fill=tk.X)
#make a separator line
sep = tk.Frame( self.root, height = 2, width = self.initDx, bd = 1, relief = tk.SUNKEN)
sep.pack(side=tk.BOTTOM, padx=2, pady = 2, fill = tk.X)
#create a string variable to access later when we the user clicks on a data
self.locationVar = tk.StringVar()
self.locationVar.set('Point')
tk.Label(self.infoFrame, textvariable = self.locationVar).pack()
return
# This will create the axes on the canvas, based on the data stored in thew ViewObj
def buildAxes(self):
self.vtm = self.viewObj.build()
# point location matrix
self.points = np.matrix([ [ 0,1,0,0,0,0],
[ 0,0,0,1,0,0],
[ 0,0,0,0,0,1],
[ 1,1,1,1,1,1]])
self.axes = self.vtm*self.points
self.lines = []
#draw axes
# x-axis
self.lines.append(self.canvas.create_line(self.axes[0,0],self.axes[1,0],self.axes[0,1],self.axes[1,1], fill="red"))
# y-axis
self.lines.append(self.canvas.create_line(self.axes[0,2],self.axes[1,2],self.axes[0,3],self.axes[1,3], fill="blue"))
# z-axis
self.lines.append(self.canvas.create_line(self.axes[0,4],self.axes[1,4],self.axes[0,5],self.axes[1,5], fill="yellow"))
self.canvas.pack()
def buildLabels(self):
if( self.xlabel != None):
#delete old labels
self.canvas.delete(self.xlabel)
self.canvas.delete(self.ylabel)
self.canvas.delete(self.zlabel)
#draw labels
self.labels = np.matrix([ [ 0.5, 1, 0, 0, 0, 0],
[ 0 , 0, 0.5, 1, 0, 0],
[ 0 , 0, 0, 0, 0.5, 1],
[ 1 , 1, 1, 1, 1, 1]])
offset = self.viewObj.getOffset()
#bring the x axis label back to about halfway between the bottom axes and the end
# of the plot
#LabelTranslate = self.viewObj.translate(-offset[0,0], 0, 0)
#xlabels = LabelTranslate*self.vtm*self.labels
self.vtm = self.viewObj.build()
labels = self.vtm*self.labels
self.xlabel = self.canvas.create_text(labels[0,0] ,labels[1,0] + offset[0,1]/2 , text = self.xAxisLabel.get())
self.ylabel = self.canvas.create_text(labels[0,2]-offset[0,0]/2,labels[1,2] + offset[0,1]/2 , text = self.yAxisLabel.get())
self.zlabel = self.canvas.create_text(labels[0,4]-offset[0,0]/2,labels[1,4] + offset[0,1]/2 , text = self.zAxisLabel.get())
self.canvas.pack()
# this will update the positions of the axes based on the data in the viewObj (i.e. creates a VTM)
def updateAxes(self):
# if the axes have not been created yet (picture is still there), do nothing
if( len(self.lines) == 0):
return
#build the vtm
self.vtm = self.viewObj.build()
#multiply the axis endpoints by the vtm
self.axes = self.vtm*self.points
# for each line object
#update the coordinates of the object
self.canvas.coords(self.lines[0],self.axes[0,0],self.axes[1,0],self.axes[0,1],self.axes[1,1] )
self.canvas.coords(self.lines[1],self.axes[0,2],self.axes[1,2],self.axes[0,3],self.axes[1,3] )
self.canvas.coords(self.lines[2],self.axes[0,4],self.axes[1,4],self.axes[0,5],self.axes[1,5] )
def updateLabels(self):
#if the labels have not yet been created (picture is still there), do nothing
if( self.xlabel == None):
return
offset = self.viewObj.getOffset()
self.vtm = self.viewObj.build()
labels = self.vtm*self.labels
self.canvas.coords(self.xlabel, labels[0,0] , labels[1,0]+ offset[0,1]/2)
self.canvas.coords(self.ylabel, labels[0,2]-offset[0,0]/2, labels[1,2]+ offset[0,1]/2)
self.canvas.coords(self.zlabel, labels[0,4]-offset[0,0]/2, labels[1,4]+ offset[0,1]/2)
# this method builds the menu items that will be accessible for the user when the display is open
def buildMenus(self):
# create a new menu
self.menu = tk.Menu(self.root)
# set the root menu to our new menu
self.root.config(menu = self.menu)
# create a variable to hold the individual menus
self.menulist = []
# create a file menu
filemenu = tk.Menu( self.menu )
self.menu.add_cascade( label = "File", menu = filemenu )
self.menulist.append(filemenu)
# create another menu for kicks
cmdmenu = tk.Menu( self.menu )
self.menu.add_cascade( label = "Command", menu = cmdmenu )
self.menulist.append(cmdmenu)
# menu text for the elements
menutext = [ [ 'Open...', '-', 'Quit \xE2\x8C\x98-Q' ],
[ 'Cluster \xE2\x8C\x98-C', 'Clean Data', 'PCA', 'Filter Data', 'Count Occurances', 'Print Data' ] ]
# menu callback functions
menucmd = [ [self.handleOpen, None, self.handleQuit],
[self.handleCmd1, self.handleCmd2, self.handleCmdButton4, self.filterData,self.countData, self.printData] ]
# build the menu elements and callbacks
for i in range( len( self.menulist ) ):
for j in range( len( menutext[i]) ):
if menutext[i][j] != '-':
self.menulist[i].add_command( label = menutext[i][j], command=menucmd[i][j] )
else:
self.menulist[i].add_separator()
# create the canvas object
def buildCanvas(self):
# this makes the canvas the same size as the window, but it could be smaller
self.canvas = tk.Canvas( self.root, width=self.initDx, height=self.initDy )
#expand it to the size of the window and fill
self.canvas.pack( expand=tk.YES, fill=tk.BOTH)
return
# build a frame and put controls in it
def buildControls(self):
# make a control frame
self.cntlframe = tk.Frame(self.root)
self.cntlframe.pack(side=tk.RIGHT, padx=2, pady=2, fill=tk.Y)
# make a separator line
sep = tk.Frame( self.root, height=self.initDy, width=2, bd=1, relief=tk.SUNKEN )
sep.pack( side=tk.RIGHT, padx = 2, pady = 2, fill=tk.Y)
# make a cmd 1 button in the frame
self.buttons = []
#width should be in characters. stored in a touple with the first one being a tag
# this button will open a display for the user to plot the data
self.buttons.append( ( 'cmd3', tk.Button( self.cntlframe, text="Plot", command=self.handleCmdButton3, width =5) ) )
self.buttons[-1][1].pack(side = tk.TOP)
# this will reset the graph image
self.buttons.append( ( 'cmd1', tk.Button( self.cntlframe, text="Reset", command=self.handleCmdButton, width=5 ) ) )
self.buttons[-1][1].pack(side=tk.TOP) # default side is top
# this will display the graph in a 3d type prespective
self.buttons.append( ( 'cmd2', tk.Button( self.cntlframe, text="3D", command=self.handleCmdButton2, width=5 ) ) )
self.buttons[-1][1].pack(side=tk.TOP) # default side is top
#this will do the data pca
self.buttons.append( ( 'cmd4', tk.Button( self.cntlframe, text="PCA", command=self.handleCmdButton4, width=5) ) )
self.buttons[-1][1].pack(side=tk.TOP)
#this will bring up a popup dialog to filter the data
self.buttons.append( ( 'cmd5', tk.Button( self.cntlframe, text="Filter", command=self.filterData, width=5) ) )
self.buttons[-1][1].pack(side=tk.TOP)
#this will put a button on the window that will open up another dialog which will
#allow the user to count the occurances of something happening given a certain condition
self.buttons.append( ('comd7', tk.Button( self.cntlframe, text="Count", command = self.countData)))
self.buttons[-1][1].pack(side=tk.TOP)
#this will print the data to an html file
self.buttons.append( ('cmd6', tk.Button( self.cntlframe, text = "Stats", command = self.printData, width = 5)))
self.buttons[-1][1].pack(side=tk.TOP)
#this adds a label which will show you the number of points in the data set
tk.Label(self.cntlframe, text = "Points:").pack()
self.Points = StringVar()
self.Points.set("-")
tk.Label(self.cntlframe, textvariable = self.Points).pack()
#this adds the number of variables so that the user can view it
tk.Label(self.cntlframe, text = "Variables:").pack()
self.Variables = StringVar()
self.Variables.set("-")
tk.Label(self.cntlframe, textvariable = self.Variables).pack()
return
# This method will update the "number of points" stringVar
# which displays the number of points in the current data set( as in filtered) on the
# right of the window for the user
def upDatePoints(self):
numPoints = self.dataObject.size()
self.Points.set(str(numPoints))
return
# This method will update the "number of variables" stringVar
# which displays the number of variables on the right of the window
def upDateVariables(self):
variables = self.dataObject.dim()
self.Variables.set(str(variables))
return
#This button will print out the data of the current data set to an
#html file and then open it
def printData(self):
#check to see if the data is loaded
test = self.checkData()
if( test == False):
return
filename = StringVar()
filename.set("stats.html")
temp = getInput(parent = self.root, inputText = "filename:", entryVar = filename, title = "Choose filename to save data")
#if the user did not input any filename
if( filename.get() == ""):
return
check = self.dataObject.printDataHtml(filename.get(), filterDataBoolean = False, countStats = True)
#check to see if the data was saved
if( check == 1):
#open the html file in a browser
os.system("open " + filename.get())
# This will set all the bindings for the different mouse/button clicks
def setBindings(self):
self.canvas.bind( '<Button-1>' , self.handleButton1 )
self.canvas.bind( '<Button-2>' , self.handleButton2 )
self.canvas.bind( '<Button-3>' , self.handleButton3 )
self.canvas.bind( '<B1-Motion>', self.handleButton1Motion )
self.canvas.bind( '<ButtonRelease-1>', self.handleButton1Release)
self.canvas.bind( '<B2-Motion>', self.handleButton2Motion )
self.canvas.bind( '<B3-Motion>', self.handleButton3Motion )
self.root.bind( '<Command-q>', self.handleModQ )
self.root.bind( '<Command-o>', self.handleModO )
self.root.bind( '<Command-t>' , self.countData)
self.root.bind( '<Control-q>', self.handleQuit )
self.root.bind( '<Control-c>', self.handleCmd1 )
self.root.bind( '<Control-f>', self.filterData )
self.canvas.bind( '<Configure>', self.resize)
self.canvas.bind( '<Control-Button-1>', self.handleButton4)
self.canvas.bind( '<Command-Button-1>', self.handleButton5)
# This method will build the pca data graph based on the information from the pca analaysis (called when you click on the
# pca button)
def buildPCA(self):
#reset size and color dim which are not used in this plot section
self.size = None
self.Color = None
# delete the axes
for i in range(len(self.objects)):
self.canvas.delete(self.objects[i])
self.objects = []
#get x,y,z data points
x = (self.pcadataHeader.get('x'))
y = (self.pcadataHeader.get('y'))
z = self.pcadataHeader.get('z')
data1 = self.Analysis.select([x,y])
#length of data
leng = self.Analysis.size()
#check to see if there is z data
if( z == None):
zData = np.zeros([leng, 1], float)
data1 = np.hstack((data1,zData))
else:
#obtain z data from the dataObject
zData = self.Analysis.select([z])
data1 = np.hstack( (data1,zData) )
#normalize matrix build
max_values = []
min_values = []
max_values.append( data1[:,0].max())
min_values.append( data1[:,0].min())
max_values.append( data1[:,1].max())
min_values.append( data1[:,1].min())
max_values.append( data1[:,2].max())
min_values.append( data1[:,2].min())
if( z == None):
scale = self.viewObj.scale(1/(max_values[0]-min_values[0]), 1/(max_values[1]-min_values[1]), 0)
else:
scale = self.viewObj.scale(1/(max_values[0]-min_values[0]), 1/(max_values[1]-min_values[1]), 1/(max_values[2]-min_values[2]))
translate = self.viewObj.translate(-min_values[0], -min_values[1], -min_values[2])
self.normmatrix = scale*translate
#add a row of 1 to every data point
hom = np.zeros([leng,1],float)
#creates a column of just 1s
hom = hom + 1;
data1 = np.hstack((data1,hom));
self.data_num = data1
#build vtm
vtm = self.viewObj.build()
#plot the data on the graph.
rowNum = 0
for row in self.data_num:
point = vtm * self.normmatrix * row.T
#standard size
dx = self.dxDefault
#standard color
rgb = "#%02x%02x%02x" % (0, 0, 0)
#create and add oval
oval = self.canvas.create_oval( point[0,0]-dx, point[1,0]-dx, point[0,0]+dx, point[1,0]+dx, fill = rgb, outline = '')
self.objects.append(oval)
rowNum += 1
def clearPoints(self):
# delete the axes
for i in range(len(self.objects)):
self.canvas.delete(self.objects[i])
self.objects = []
#reset axes labels
self.xAxisLabel.set('x')
self.yAxisLabel.set('y')
self.zAxisLabel.set('z')
self.buildLabels()
# Puts points on the canvas
def buildPoints(self):
# delete the axes
for i in range(len(self.objects)):
self.canvas.delete(self.objects[i])
self.objects = []
self.size = None
self.Color = None
self.data_num = None
#get the x,y,z,color,size header names if they exist
x = self.headerData.get('x')
y = self.headerData.get('y')
z = self.headerData.get('z')
color = self.headerData.get('color')
sizeV = self.headerData.get('size')
#if there is only one input, then make a histogram
if( x != '' ):
if( y == ''):
pylab.ion()
pylab.cla()
#print self.dataObject.select([x])
pylab.hist(self.dataObject.select([x]))
pylab.xlabel(x)
pylab.ylabel("Occurances")
pylab.show()
return
#get data
data1 = self.dataObject.select([x,y])
#if the data is empty exit this method
if( np.shape(data1)[1] == 0):
print "No Data, remove filters or change file!"
return
#length of data
leng = self.dataObject.size()
#check to see if there is z data
if( z == None):
zData = np.zeros([leng, 1], float)
data1 = np.hstack((data1,zData))
else:
#obtain z data from the dataObject
zData = self.dataObject.select([z])
data1 = np.hstack( (data1,zData) )
#add a row of 1 to every data point
hom = np.zeros([leng,1],float)
#creates a column of just 1s
hom = hom + 1;
data1 = np.hstack((data1,hom));
self.data_num = data1
#check to see if there is a color data
if( color != None):
colorData = self.dataObject.select([color])
#add homogeneous coordinate
colorData = np.hstack((colorData,hom))
self.color = colorData
else:
self.color = None
#check to see if there is a size data
if( sizeV != None):
sizeData = self.dataObject.select([sizeV])
#add homogeneous coordinate
sizeData = np.hstack((sizeData,hom))
self.size = sizeData
else:
self.size = None
#the code below is for data normalization using the (x-xMin)/(xMax-xMin) process for each
#dimension of code
#xy ranges
xRange = self.dataObject.range_num(self.dataObject.getIndex(x))
yRange = self.dataObject.range_num(self.dataObject.getIndex(y))
# z range which is optional
if( z != None):
zRange = self.dataObject.range_num(self.dataObject.getIndex(z))
#scale matrix to divide by (xMax-xMin)
scale = self.viewObj.scale(1/(xRange[0]-xRange[1]), 1/(yRange[0]-yRange[1]), 1/(zRange[0]-zRange[1]))
#no z, so it is all zeros and can be just left unnormalized
else:
zRange = [0,0]
#scale matrix to divide by (xMax-xMin)
scale = self.viewObj.scale(1/(xRange[0]-xRange[1]), 1/(yRange[0]-yRange[1]), 0)
#create the translation matrix to do the x-xMin part
translate = self.viewObj.translate(-xRange[1], -yRange[1], -zRange[1])
#create the matrix that will do the normalization of the data (i.e. (x - xMin)/(xMax-xMin))
self.normmatrix = scale*translate
#if there is a color category
if( color != None):
# normalize the color scale information
colorMinMax = self.dataObject.range_num(self.dataObject.getIndex(color))
self.colorNorm = (self.color - colorMinMax[1])/(colorMinMax[0]-colorMinMax[1])
# if there is a size section, normalize it
if( sizeV != None):
sizeMinMax = self.dataObject.range_num(self.dataObject.getIndex(sizeV))
self.sizeNorm = (self.size - sizeMinMax[1])/(sizeMinMax[0] - sizeMinMax[1])
#build the vtm
vtm = self.viewObj.build()
#plot the data on the graph.
rowNum = 0
#for each data vector we need to normalize it and then apply the vtm to get the screen coordinates
for row in self.data_num:
point = vtm * self.normmatrix * row.T
#handles the size dimension
if( sizeV != None):
#somewhere between min and max, there is a slight cutoff at the min just so points are visible
dx = self.sizeNorm[rowNum,0]*(self.dxMax)
if( dx < self.dxMin):
dx = self.dxMin
#set the default size if no size dim
else:
dx = self.dxDefault
#handles the color dimension
if( color != None):
#get the type of the data (should be either numeric or enum)
type = self.dataObject.getType(self.dataObject.getIndex(color))
#since it is normalized between 0,1 this is a direct correlation
alpha = self.colorNorm[rowNum,0]
#base it off of the HSV color scale
if(type == 'enum'):
#multiply alpha by 0.8 to prevent going all the way around
rgb = colorsys.hsv_to_rgb(alpha*0.8, 1, 1)
rgb = (int(float(rgb[0])*200), int(float(rgb[1])*200), int(float(rgb[2])*200))
rgb = "#%02x%02x%02x" % (rgb[0], rgb[1], rgb[2])
else:
rgb = "#%02x%02x%02x" % (200*alpha, 0.0, (1-alpha)*200)
#default color == black
else:
rgb = "#%02x%02x%02x" % (0, 0, 0)
# this part stores the location of the point in data space to the oval so that it can be accessed later when we only know
# the screen coordinates.
if( z != None):
Location = '%s: %.2f , %s: %.2f , %s: %.2f' % ( x, row[0,0], y, row[0,1], z, row[0,2] )
else:
Location = '%s: %.2f , %s: %.2f' % ( x, row[0,0], y, row[0,1] )
#check to see if size and color are being used
if( sizeV != None):
Location += ', %s: %.2f' % (sizeV, self.size[rowNum,0])
if( color != None):
Location += ', %s: %.2f' % (color, self.color[rowNum,0])
#add in the row number
Location += ' %d' % (rowNum)
#create and add oval
oval = self.canvas.create_oval( point[0,0]-dx, point[1,0]-dx, point[0,0]+dx, point[1,0]+dx, fill = rgb, outline = '', tag = Location)
self.objects.append(oval)
rowNum += 1
# This method will update all of the points in the canvas graph based on the vtm in the viewObj. Color dim does not need to be updated because
# it is never reset
def updatePoints(self):
if( len(self.objects) == 0):
return
# if there are data objects plotted
if ( self.data_num != None):
vtm = self.viewObj.build()
rowNum = 0
#for each row in the numeric data
for row in self.data_num:
point = vtm *self.normmatrix* row.T
#handles the size dimension
if( self.size != None):
#somewhere between min and max, there is a slight cutoff at the min just so points are visible
dx = self.sizeNorm[rowNum,0]*(self.dxMax)
if( dx < self.dxMin):
dx = self.dxMin
#set the default size if no size dim
else:
dx = self.dxDefault
#color dim does not need to change here!
#adjust positions
self.canvas.coords(self.objects[rowNum],point[0,0]-dx, point[1,0]-dx, point[0,0]+dx, point[1,0]+dx)
rowNum += 1
# this method will load in the data. It does not plot the data and only loads the data into the programs memory
def handleOpen(self):
print 'handleOpen'
#open file that has the data
fobj = tkFileDialog.askopenfile( parent=self.root, mode='rU', title='Choose a data file (.csv)' )
#error checking
if( fobj == None):
return False
self.dataObject = csvDataReader(pathName = fobj.name)
self.upDatePoints()
self.upDateVariables()
print "Data Loaded!"
#this returns the axes to the default location and then builds the labels and axes
self.returnDefault()
if( len(self.lines) == 0):
self.buildAxes()
#deletes the image if it is still on the canvas
self.canvas.delete(self.imageWidget)
self.clearPoints()
return True
def handleQuit(self, event=None):
print 'Terminating'
self.root.destroy()
def handleModQ(self, event):
self.handleQuit()
def handleModO(self, event):
self.handleOpen()
# This method tests to see if there is any data loaded
# if there is no data loaded it will prompt the user to load data.
# It will then return True if there is data loaded (either before or during the method)
# or False if there is no data loaded
def checkData(self):
# no dataObject
if( self.dataObject == None):
print "Open data file"
#create dialog box
response = tkMessageBox.askyesno(title = "No Loaded Data!", message = "Load data?")
#if the user wants to laod data
if response:
check = self.handleOpen()
if(check == False):
return False
return True
#No data loaded, return false
else:
return False
else:
return True
# this method will reset the view object to the default settings
# for the viewer.
def returnDefault(self):
self.viewObj.setExtent(np.matrix([1,1,1]))
self.viewObj.setOffset(np.matrix([20,20]))
#self.viewObj.setScreen(np.matrix([400,400]))
self.viewObj.setScreen(np.matrix([self.canvas.winfo_width(),self.canvas.winfo_height()]) - self.windowOffset)
self.viewObj.setU(u = np.matrix([1,0,0]))
self.viewObj.setVPN(vpn = np.matrix([0,0,-1]))
self.viewObj.setVRP(vrp = np.matrix([0.5,0.5,1]))
self.viewObj.setVUP(vup = np.matrix([0,1,0]))
self.updateAxes()
self.updatePoints()
self.updateLabels()
return
#reset to default
def handleCmdButton(self):
print 'handling command button'
self.returnDefault()
return
# 3d view for the user
def handleCmdButton2(self):
test = self.checkData()
if( test == False):
return
print 'handling command button2'
self.viewObj.setExtent(np.matrix([ 1.96429238, 1.96429238, 1.96429238]))
self.viewObj.setOffset(np.matrix([20,-20]))
#self.viewObj.setScreen(np.matrix([400,400]))
self.viewObj.setScreen(np.matrix([self.canvas.winfo_width(),self.canvas.winfo_height()]))
#print self.viewObj.getScreen()
self.viewObj.setU(u = np.matrix([-0.89773056, 0.10085255, -0.42884566]))
self.viewObj.setVPN(vpn = np.matrix([ 0.44036241, 0.23344776, -0.86693892]))
self.viewObj.setVRP(vrp = np.matrix([ 0.43337459, 0.56723291, 0.76298161]))
self.viewObj.setVUP(vup = np.matrix([-0.01268006, 0.96712507, 0.25398486]))
self.updateAxes()
self.updateLabels()
self.updatePoints()
return
# this method will open up the plot dialog for the user so that it can determine which categories in the data
# to plot.
def handleCmdButton3(self):
#check to see if there is data loaded
test = self.checkData()
if( test == False):
return
print 'handling command button3'
if( self.dataObject.filtersize() == 0):
tkMessageBox.showwarning("No data!","Remove Filters from the data or load a different data set!")
return
#create a new dictionary that will store the headers for the x,y,z,size, and color dim
self.headerData = {}
#open input dialog
inputBox = MyDialog(parent = self.root,displayClass = self, dataHeader = self.headerData)
if(self.headerData['check'] == True):
#plot the points
self.buildPoints()
#check points
self.xAxisLabel.set( self.headerData.get('x') )
self.yAxisLabel.set( self.headerData.get('y') )
#z is an optional component
z = self.headerData.get('z')
if( z == None ):
z = ""
self.zAxisLabel.set( z )
self.buildLabels()
#This mehtod will open up a dialog for the user so that it can determine which categories it wants to plot after
# PCA analysis has been computed on the data set
def handleCmdButton4(self):
#check to see if data has been loaded
test = self.checkData()
if( test == False):
return
self.size = None
self.color = None
#check to see if there is any data in the set after filters applied
if( self.dataObject.filtersize() == 0):
tkMessageBox.showwarning("No data!","Remove Filters from the data or load a different data set!")
return
print "handling command button 4"
# run pca on all of the numerical data
self.Analysis = Analysis(name = "PCA", data = self.dataObject.getData_num())
#dictioanry that will store the headers for the pca analsysis plots
self.pcadataHeader = {}
# this will run the dialog for pca
inputBox = PCADialog(parent = self.root, pcaDataHeader = self.pcadataHeader, displayClass = self, Analysis = self.Analysis)
if( self.pcadataHeader['check'] == True):
self.xAxisLabel.set( self.pcadataHeader.get('x') )
self.yAxisLabel.set( self.pcadataHeader.get('y') )
z = self.pcadataHeader.get('z')
#z is optional
if( z == None ):
z = ""
self.zAxisLabel.set(z)
self.buildLabels()
#graphs the pca points
self.buildPCA()
if( self.pcadataHeader.get('saveData') ):
filename = self.pcadataHeader.get('filename')
filename = filename + ".csv"
self.Analysis.writeData(filename = filename)
#This method will open up the filter choice menu dialog where the user will
#be able to choose which fitlers to add/remove from the data.
#this will also have an option for removing missing data.
def filterData(self, event=None):
#check to see if the data is loaded
test = self.checkData()
if( test == False):
return
a = filterDialog(parent = self.root, displayClass = self, dataObj = self.dataObject, title = "Filter")
return
# This method will open the countInfo Dialog box where the user can count the number
# of data points that meet two conditions which will be saved in the data object
def countData(self, event = None):
#check to see if the data is loaded
test = self.checkData()
if( test == False):
return
#open dialog
temp = countInfo(parent = self.root, dataObject = self.dataObject, title = "Count Occurrences within the Data:")
return
# this method will run the kmeans clustering algorithm on the loaded data
# a popup dialog will appear with options for the user to choose which columns
# to run on the kmeans clustering. Default being use all numeric columns
# the user can also choose the number of clusters to have and the name
# of this cluster run
def handleCmd1(self, event = None):
#check to see if the data is loaded
test = self.checkData()
if( test == False):
return
#check to see if there is any data
if( self.dataObject.filtersize() == 0):
tkMessageBox.showwarning("No data!","Remove Filters from the data or load a different data set!")
return
#obtain data
self.clusterInfo = {}
clusterDialogBox = ClusterDialog(parent = self.root, displayClass = self, clusterInfo = self.clusterInfo,title="Clusters")
#if nothing is chosen do nothing and exit
if( self.clusterInfo.get("headers") == None):
return
headers = self.clusterInfo.get("headers")
self.dataObject.cluster(headers = headers, clusters = self.clusterInfo.get("clusters"), name = self.clusterInfo.get("name"), normalize = True)
tkMessageBox.showwarning("Clustering Complete", "The clustering process has now completed!")
print 'handling command 1'
return
#this method will be used to clean data
def handleCmd2(self):
#check to see if the data is loaded
test = self.checkData()
if( test == False):
return
#initialize filename variable
filename = StringVar()
temp = getInput(parent = self.root, inputText = "Choose Filename:", entryVar = filename, title = "Clean data of missing information")
#if nothing is input
if( filename.get() == ""):
return
#clean the data
self.dataObject.setMissingToAverage()
#save it
self.dataObject.saveData(filename.get())
#load the new data back into the system
self.dataOBject = csvDataReader(pathName = filename.get())
self.upDatePoints()
self.upDateVariables()
print "Data Loaded!"
return
def handleCmd3(self):
print 'handling command 3'
# Stores scaling data initial values
def handleButton3(self, event):
print "handling button3"
self.baseClick3 = (event.x, event.y)
self.textent = self.viewObj.getExtent()
return
#scales the values of the axes/data based on y movement when button 3 is depressed, all based off of initial click location
def handleButton3Motion(self, event):
diff = event.y-self.baseClick3[1]
#determine scale
scale = 1+(3.0*diff)/self.root.winfo_height()
#print "scale",scale
#check to see extremes of scale
if( scale < 0.1):
scale = 0.1
elif( scale > 3):
scale = 3
#change the extent
self.viewObj.setExtent(self.textent*(scale))
#update drawings
self.updateAxes()
self.updateLabels()
self.updatePoints()
# store initial value for translation
def handleButton1(self, event):
#print 'handle button 1: %d %d' % (event.x, event.y)
self.baseClick1 = (event.x, event.y)
closestObj = None
#for obj in self.objects:
# store initial values before rotation (i.e. clone view object)
def handleButton2(self, event):
#print 'handle button 2: %d %d' % (event.x, event.y)
self.baseClick2 = (event.x,event.y)
self.viewObjClone = self.viewObj.copy(View())
return
# rotates the data based on movement from initial click location. 400 px is about 1 full rotation
def handleButton2Motion(self, event):
divConst = 200.0
#rotation angles
delta0 = ((event.x-self.baseClick2[0])/divConst)*math.pi
delta1 = ((event.y-self.baseClick2[1])/divConst)*math.pi
#clone view object
viewObjClone2 = self.viewObjClone.copy(View())
#rotate
viewObjClone2.rotateVRC(delta0,delta1)
# update points, axes, view obj
self.viewObj = viewObjClone2
self.updateAxes()
self.updateLabels()
self.updatePoints()
return
# This is executed when the user releases button1 of the mouse and starts the callback
# for the continual "inertial" motion
def handleButton1Release(self, event):
#call my callback method
self.callback_Motion()
# This call back method will continue to be called and apply the inertial effects of the tranlsation
# caused by the user until it reaches some minimum. The constants self.kn and self.check are set in the
# constructor method. This method will be called by the program every 20 ms to update the axes
def callback_Motion(self):
# update our travel vector by some scalar constant self.kn
self.Delta = self.Delta*self.kn
# this checks to see if the change is between -check < x < check for x,y,z variables
if( self.Delta[0,0] < self.check and self.Delta[0,0] > -self.check):
if( self.Delta[0,1] < self.check and self.Delta[0,1] > -self.check):
if( self.Delta[0,2] < self.check and self.Delta[0,2] > -self.check):
# if it is exit and end callbacks
return
# update the vrp location
vrp = self.viewObj.getVRP()
vrp = vrp + self.Delta
self.viewObj.setVRP(vrp)
#update points and axes
self.updateAxes()
self.updateLabels()
self.updatePoints()
#set callback method call
self.root.after(20, self.callback_Motion)
return
# this method will handle the 'control-mouse click' combo which will display the point location in data space of the nearest point
# to the user's click
def handleButton4(self,event):
#mouse click location
x = event.x
y = event.y
#gets id of closest oval
closest = self.canvas.find_closest(x, y, halo = None, start = None)
# this will get the tag of the closest oval which stores the location in data space of the point
tag = self.canvas.gettags(closest[0])
# this creates the pointInfo string which will be put on the display
pointInfo = ''
for i in range(0,len(tag)-2):
pointInfo += " " + tag[i]
#update display
self.locationVar.set("Point: " + pointInfo)
return
# this method will handle the 'command-mouse click' combo which will open a display dialog which will display
# all of the data for that data vector in an image
def handleButton5(self,event):
#mouse click location
x = event.x
y = event.y
#gets id of closest oval
closest = self.canvas.find_closest(x, y, halo = None, start = None)
# this will get the tag of the closest oval which stores the location in data space of the point
tag = self.canvas.gettags(closest[0])
length = len(tag)
if(tag[length-1] == 'current'):
rowNum = tag[length-2]
else:
rowNum = tag[length-1]
#rowNum = tag[length-2]
#print tag
#print rowNum
dialog = InfoBox(parent = self.root, dataObject = self.dataObject, rowNumber = rowNum, title = "Data Information")
return
# handles translation of the axes / data
def handleButton1Motion(self, event):
# this creates a differential motion information for us by updating this
# every time the mouse is moved
# calculate the difference
diff = ( event.x - self.baseClick1[0], event.y - self.baseClick1[1] )
self.baseClick1 = (event.x, event.y)
norm = []
norm.append( float(diff[0])/self.root.winfo_width())
#print "width", self.root.winfo_width()
norm.append( float(diff[1])/self.root.winfo_height())
#print "norm",norm
adj = 0.8
#change amount
delta0 = adj*norm[0]*self.viewObj.getExtent()[0,0]
delta1 = adj*norm[1]*self.viewObj.getExtent()[0,1]
# adjust vrp
vrp = self.viewObj.getVRP()
self.Delta = delta0*self.viewObj.getU() + delta1*self.viewObj.getVUP()
vrp = vrp + self.Delta
self.viewObj.setVRP(vrp)
#redraw points
self.updateAxes()
self.updateLabels()
self.updatePoints()
# handles if if the canvas size is changed, will reset the data
def resize(self,event):
#self.viewObj.setScreen(np.matrix([self.canvas.winfo_width(), self.canvas.winfo_height()]))
#self.handleCmdButton()
#self.updateAxes()
#self.updatePoints()
self.returnDefault()
return
def main(self):
print 'Entering main loop'
#lets everything just sit and listen
self.root.mainloop()
def initiate_analysis(self):
# Load input data from tables
section = self.get_geometry()
SF = self.getSF()
Mat = self.get_material()
# check if geometry is valid
valid, msg = section.valid()
if not valid:
self.show_msg_box(['The defined geometry is not valid', msg])
self.Res = None
self.refresh_visible_plots()
return
print(section)
print(Mat)
print(SF)
# switch to Loading & Result tab
self.tabWidget.setCurrentIndex(3)
# set mouse cursor to WaitCursor
self.setCursor(QtCore.Qt.WaitCursor)
try:
# Call analysis
if self.checkBox_analSLS_1.isChecked():
# execute SLS analysis
string = self.checkBox_analSLS_1.text()
self.statusbar.showMessage(string + ' analysis initiated')
self.Res = Analysis.SLS_analysis(section, SF, Mat)
error_msg = None # errors are now passed as exceptions
self.load_fac_label.setText('No load-factor currently applied'
) # <-- might not be needed
self.statusbar.showMessage(string + ' analysis completed')
elif self.checkBox_analULS_1.isChecked():
# execute ULS analysis
string = self.checkBox_analULS_1.text()
self.statusbar.showMessage(string + ' analysis initiated')
self.Res, error_msg = Analysis.ULS_analysis(section, SF, Mat)
self.statusbar.showMessage(string + ' analysis completed')
else:
self.Res = None
error_msg = 'No analysis method is checked'
self.load_fac_label.setText('')
except Analysis.MyOptimizerError as e:
# caught a MyOptimizerError exception
self.show_msg_box([str(e), e.discription])
error_msg = None
else:
# no MyOptimizerError exception
self.load_fac_label.setText('No load-factor currently applied')
# Show message
if error_msg:
self.show_msg_box(error_msg, set_load_fac_label=True)
else:
self.load_fac_label.setText('No load-factor currently applied')
# update result plot
self.refresh_visible_plots()
# return mouse cursor to normal ArrowCursor
self.setCursor(QtCore.Qt.ArrowCursor)
def __init__(self, data, filename, view, parent):
super(DisassemblerView, self).__init__(parent)
self.status = ""
self.view = view
self.data = data
for type in ExeFormats:
exe = type(data)
if exe.valid:
self.data = exe
self.view.exe = exe
break
# Create analysis and start it in another thread
self.analysis = Analysis(self.data)
self.analysis_thread = threading.Thread(None,
self.analysis_thread_proc)
self.analysis_thread.daemon = True
self.analysis_thread.start()
# Start disassembly view at the entry point of the binary
if hasattr(self.data, "entry"):
self.function = self.data.entry()
else:
self.function = None
self.update_id = None
self.ready = False
self.desired_pos = None
self.highlight_token = None
self.cur_instr = None
self.scroll_mode = False
self.blocks = {}
self.show_il = False
self.simulation = None
# Create timer to automatically refresh view when it needs to be updated
self.updateTimer = QTimer()
self.updateTimer.setInterval(100)
self.updateTimer.setSingleShot(False)
self.updateTimer.timeout.connect(self.updateTimerEvent)
self.updateTimer.start()
self.initFont()
# Initialize scroll bars
self.width = 0
self.height = 0
self.setHorizontalScrollBarPolicy(Qt.ScrollBarAsNeeded)
self.setVerticalScrollBarPolicy(Qt.ScrollBarAsNeeded)
self.horizontalScrollBar().setSingleStep(self.charWidth)
self.verticalScrollBar().setSingleStep(self.charHeight)
areaSize = self.viewport().size()
self.adjustSize(areaSize.width(), areaSize.height())
# Setup navigation
self.view.register_navigate("disassembler", self, self.navigate)
self.view.register_navigate("make_proc", self, self.make_proc)
self.search_regex = None
self.last_search_type = FindDialog.SEARCH_HEX
def quasi_airy(center_x, center_y, width_x, width_y, offset, height):
def quairy_func(x,y):
R = sqrt((x - center_x)**2/width_x + (y - center_y)**2 / width_y)
return offset + height * jn(1, R)/(R+1e-6)
return quairy_func
if __name__ == '__main__':
funcs = {"Quasi-Airy": quasi_airy, "Gaussian": Analysis.gaussian, \
"Airy" : Analysis.airy, "Bessel", bessel}
rmses = {}
for i in funcs.keys():
rmses[i] = 0
try:
os.uname()
dirname = '/Users/pcombs/Documents/RotationData/2009-12-08-FITS/*'
bgname = '/Users/pcombs/Documents/RotationData/2009-12-08/MED_2009-12-08.fits'
except AttributeError:
dirname = r'C:\Documents and Settings\admin\Desktop\Peter\2009-12-09-FITS/8*'
bgname = r'C:\Documents and Settings\admin\Desktop\Peter\MED_2009-12-08.fits'
filelist = glob(dirname)
filelist = [os.path.join(dirname, f) for f in filelist]
filelist = filelist
for func_name in funcs.keys():
fit = Analysis.fit2d(xs, ys, data, fcn = funcs[func_name])
import os
import Image_Process
import cv2
import cv2.cv as cv
import numpy as np
import BoW
import SVM
import Analysis
import Descriptors
B = BoW.BoW("BoWSS.pkl")
# if B.loaded:
# print "BoW already computed"
# exit
path = "/media/iglu/Data/rgbd-dataset"
i = 0
Analysis.initial()
start_time = timeit.default_timer()
i = 0
for x in os.listdir(path):
# if i == 0:
# i += 1
# continue
# else:
mn=0
D=False
R=False
if x.startswith("Test"):
continue
for y in os.listdir(path + "/" + x):
for z in os.listdir(path +"/"+x +"/"+y+"/"):
if z.endswith("_crop.png"):
import numpy as np
import matplotlib.pyplot as plt
import Analysis as anl
""""
get data from NNGT saved files_double_event
"""
rootpath = "/home/mallory/Documents/These/javier-avril-CR/Simulations2/GaussianNetworks/IBneurons/"
filename = "2000Gaussian_30_6.0_weight166.67withminis.txt"
tmin = 0.
tmax = 200000.
raster = anl.load_raster(rootpath + filename,
tmin=tmin,
tmax=tmax,
with_space=False)
"""
Find the bursts : ce passage prend du temps, tu peux le passer si tu
veux juste la phase et ses stat.
"""
plot_phase = True
#Choose time step in ms
step = 5.
# Ici je change les tmin et tmax parce que la phase n'est definit que
# entre deux spike, donc je regarde uniquement à ces temps la.
tmin = np.nanmin(raster[:, 0])
tmax = np.nanmax(raster[:, -1])
title=
f'Final predictions from ARIMA model for "{stock_name}" stock',
color=["#c6e2ff", "#deaddd"],
linewidth=1.5)
if save_plot:
plt.savefig('./demonstration_images/arima_predictions.png')
plt.show()
plt.figure(figsize=(12, 8))
sns.kdeplot(res["Residuals"], color="#c7c6ff", shade=True, linewidth=2)
plt.title(
f'Density of the residuals from the auto-regression for "{stock_name}" stock'
)
plt.xlabel('Residuals')
plt.ylabel('Density')
if save_plot:
#fig_r = ax_r.get_figure()
plt.savefig('./demonstration_images/arima_residuals.png')
plt.show()
if __name__ == '__main__':
df = Analysis.get_data('./Data/AMZN.csv')
df = df[["Close"]]
df.rename(columns={0: 'Close'}, inplace=True)
arima_model = ARIMA(0.8, 'Close')
df_c, res = arima_model.run(df, True)
arima_model.plot_results(df_c, res, 'AMZN', save_plot=True)
help="")
parser.add_option('-q', '--quiet', dest = "quiet",
action = "store_true", default = False)
parser.add_option('-o', '--output-file', dest="output", default=None)
parser.add_option('-c', '--center', dest="center", default=False,
help="For measuring colocalization distance, take as distance from"
"the average colocalization", action="store_true")
opts, args = parser.parse_args()
if not opts.map:
if opts.x1 and opts.x2 and opts.y1 and opts.y2:
opts.xshift = opts.x2 - opts.x1
opts.yshift = opts.y2 - opts.y1
else:
mapping = Analysis.loadmapping(opts.map)
if opts.map2:
mapping2 = Analysis.loadmapping(opts.map2)
timepoints = defaultdict(list)
tpmins = [0]*100
tpmaxes = [0]*100
imgname_finder = re.compile('(.*)_spot[0-9]+_[^_]+_xy(_only)?.txt')
spotnum_finder = re.compile('spot([0-9]+)_')
channel_finder = re.compile('spot[0-9]+_([0-9]{3})_xy(_only)?')
#print map(glob, args)
max_timepoint = 0
Load data, analyze, and output results.
"""
import os
import pandas
import pdb
import sys
import Analysis
import Constants
import Basketball
import Misc
if __name__ == "__main__":
tournaments = {}
analysis_options = Analysis.get_objects()
num_analysis_options = len(analysis_options)
# prompt user to select analysis method
prompt = "Please enter number of desired analysis method:\n"
for i in range(num_analysis_options):
prompt += "%s) %s\n" % (i, analysis_options[i].get_name())
user_input = raw_input(prompt)
# check user input
if user_input.isdigit() and (int(user_input) in range(num_analysis_options)):
analysis = analysis_options[int(user_input)]
else:
raise Exception("Invalid entry '%s', please enter one of following values %s." % (user_input, range(num_analysis_options)))
# prompt user for Kaggle output
kaggle_output_options = {
torch.save(net.state_dict(),
"Unet2_model_" + leg + "_min_val_loss") # save model
min_model_epoch = epoch + 1
disp = 'Epoch %d: Train loss: %.6f, Val loss: %.6f' % (
epoch + 1, running_loss / count, val_loss_temp)
write_2_log(disp)
print(disp)
val_loss.append(val_loss_temp)
train_loss.append(running_loss / count)
running_loss = 0.0
if (epoch + 1) in model_saved: # save checkpoint
model_name = "Unet2_model_" + str(epoch + 1) + "epoch_" + leg
torch.save(net.state_dict(), model_name) # save model
Analysis.analysis(model_file=model_name, valloader=valloader)
write_2_log("Best model at epoch: " + str(min_model_epoch))
write_2_log("Total training time: " +
str((time.time() - start_begining) / 3600.0))
val_loss_file = "data/results" + "/val_loss_" + str(
N_epoch) + "epoch_" + leg + ".npy"
train_loss_file = "data/results" + "/training_loss_" + str(
N_epoch) + "epoch_" + leg + ".npy"
np.save(val_loss_file, val_loss)
np.save(train_loss_file, train_loss)
print("total processing time in seconds: ", time.time() - start_begining)
print("total processing time in mins: ",
(time.time() - start_begining) / 60.0)
def polarityDashboard(origStock, stockList, compareData):
'''
Make a dashboard for the news polarity stuff
Output it to polarityDashboard.html
'''
#relevanceBar = barPlot(compareData, "NEWS_SIMILARITY")
titlePolarities, titleBias, textPolarities, textBias = Analysis.articleSentiments(
stockList)
allFigures = []
#allFigures.append([relevanceBar])
#para = Paragraph(text="On this dashboard, you can see the polarity of relevant news articles and headlines. Strongly positive or negative headlines will drive short-term volatility, whereas the polarity of articles impact long-term growth.")
#get averages of each
means = []
for oneStockTitles, oneStockTexts in zip(titlePolarities, textPolarities):
if len(oneStockTitles) > 0 and len(oneStockTexts) > 0:
means.append((statistics.mean(oneStockTitles) +
statistics.mean(oneStockTexts)) / 2.0)
if len(means) > 0:
xBarNames = compareData["stock_SYMBOL"] #the things that we're listing
polBar = figure(x_range=xBarNames,
title="Polarity Summary",
y_axis_label='Polarity',
height=300,
sizing_mode='scale_width')
polBar.vbar(x=xBarNames, top=means, width=0.9)
allFigures.append([polBar])
colors = ["blue", "green", "orange", "purple", "magenta", "cyan", "yellow"]
count = 1
for oneStock, stockTitles in zip(stockList, titlePolarities):
titleLen = list(range(1, len(stockTitles) + 1))
polarityFigure = figure(title=f"{oneStock.name} Headline Polarity",
x_axis_label='Time',
y_axis_label='Polarity',
height=300,
sizing_mode='scale_width')
polarityFigure.line(x=titleLen,
y=stockTitles,
legend=oneStock.symbol,
line_width=2,
color=colors[count])
if len(allFigures) - 1 > 0 and len(
allFigures[len(allFigures) - 1]) >= 2:
allFigures.append([polarityFigure])
else:
if len(allFigures) - 1 <= 0:
allFigures.append([polarityFigure])
else:
allFigures[len(allFigures) - 1].append(polarityFigure)
count += 1
if count > 5: count = 0
for oneStock, stockTexts in zip(stockList, textPolarities):
textLen = list(range(1, len(stockTexts) + 1))
polarityFigure = figure(title=f"{oneStock.name} Article Text Polarity",
x_axis_label='Time',
y_axis_label='Polarity',
height=300,
sizing_mode='scale_width')
polarityFigure.line(x=textLen,
y=stockTexts,
legend=oneStock.symbol,
line_width=2,
color=colors[count])
if len(allFigures) - 1 > 0 and len(
allFigures[len(allFigures) - 1]) >= 2:
allFigures.append([polarityFigure])
else:
if len(allFigures) - 1 <= 0:
allFigures.append([polarityFigure])
else:
allFigures[len(allFigures) - 1].append(polarityFigure)
count += 1
if count > 6: count = 0
grid = gridplot(allFigures)
return components(grid)
import TrialMoves as TM
import Analysis as an
if __name__=='__main__':
(lattice,graft_points,chains) = init.initialize_lattice((200,200,20),200,16,[2,3],20)
#previously 779
print len(graft_points)
count = 0
n = sys.argv[1]
alph = sys.argv[2]
#n = 5
#alph = 'f'
an.chains_to_xyz(chains, 'InitDual_'+str(n)+alph, lattice)
for i in range(0,int(n)):
rando = rnd.uniform(0,1)
if rando>.66:
(lattice, chains,total_energy, acc) = TM.cbmc(lattice,chains)
print "CBMC"
elif rando > .33:
(lattice, chains, total_energy, acc) = TM.take_empty(lattice, chains, graft_points)
print "Empty"
else:
(lattice, chains, total_energy, acc) = TM.swap(lattice, chains)
print "SWAP"
an.store_energies(total_energy, n, alph)
# Should add matrices for chemical moeity and identity in here
dcfs.getData()
# Get LA City zip codes (Web scraping)
# Get Walk Score data based on LA City zip codes (Web scraping)
print("\n * Getting LA City zip copdes and walk score data")
walkScore = classWalkScore.ClassWalkScore(args.source)
walkScore.getLAcityZip()
walkScore.getData()
# Get LA population data (API)
print("\n * Getting LA population data")
pop = classPop.ClassPop(args.source)
pop.getLApop()
# Get crime report data (API)
print("\n * Getting crime records data")
crime = classCrime.ClassCrime(args.source)
crime.getData()
# Cleaning data and data modeling
print("\n * Clean data and modeling")
cleanReferral.CleanReferral()
cleanReferral.ZipCrime()
# Analyze each dataset and combined dataset
Analysis.loadGeoJSON()
Analysis.Crime_analysis()
Analysis.DCFS_analysis()
Analysis.Combine_analysis()
Analysis.Correlation_graph()
