def handlePCAClustering(self):
print self.pcaDataObjects
print self.PCAlistbox.curselection()
if (bool(self.pcaDataObjects) == True):
curSelection = self.PCAlistbox.curselection(
) #List of items to be deleted
if (curSelection != None):
cBox = dialogs.kmeansDialogBox(self.root,
self.dataObject.get_headers())
if (cBox.cancelPress == False):
if (cBox.getVal != None):
selectVal = self.PCAlistbox.curselection()[
0] #index value of selected value
selectedPCA = self.PCAlistbox.get(selectVal)
dataObject = self.pcaDataObjects.get(selectedPCA)
if (cBox.getManhattan() == True):
analysisObj = analysis.Analysis()
self.kVal = cBox.name
headers = cBox.getVal()
d = dataObject.get_data(headers,
dataObject.get_num_rows())
codebook, codes, error = analysisObj.kmeans(
d, headers, self.kVal, manhattan=True)
dataObject.add_column(
np.matrix(codes).T, 'codes', 'code')
self.buildPCA(dataObject)
else:
analysisObj = analysis.Analysis()
self.kVal = cBox.name
headers = cBox.getVal()
d = dataObject.get_data(headers,
dataObject.get_num_rows())
codebook, codes, error = analysisObj.kmeans_numpy(
d, headers, self.kVal)
dataObject.add_column(
np.matrix(codes).T, 'codes', 'code')
self.buildPCA(dataObject)
else:
print "Please select a PCA analysis to Run K mean on"
else:
print "Please perform PCA Analysis first"
def visualizeArbitraryAOI(imageName):
analysisOb = analysis.Analysis(parameters)
analysisOb.datasets = DATASET_FOLDER
plotterOb = analysisOb.getPlotter()
pngFile = imageName.replace(".jpg",".png",-3)
plotterOb.plotArbitraryAOI(VISUALIZER_FOLDER+"_ArbitraryAOI",VISUALIZER_FOLDER+pngFile)
def visualizeParticipantsGazePlots(imageName,ParticipantNumber):
analysisOb = analysis.Analysis(parameters)
analysisOb.datasets = DATASET_FOLDER
plotterOb = analysisOb.getPlotter()
pngFile = imageName.replace(".jpg",".png",-3)
plotterOb.plotParticipantPaths(imageDataset,ParticipantNumber,VISUALIZER_FOLDER+ParticipantNumber+"_gazeplot",VISUALIZER_FOLDER+pngFile)
def visualizeHeatmapsByGender(imageName):
analysisOb = analysis.Analysis(parameters)
analysisOb.datasets = DATASET_FOLDER
plotterOb = analysisOb.getPlotter()
pngFile = imageName.replace(".jpg",".png",-3)
def inerpolate(self):
if (self.analys == None):
analysis.Analysis(self)
newobj = copy.deepcopy(self)
gLen = len(self.arr['baseArr'])
newobj.arr['baseArr'] = []
for i in range(gLen - 1):
avg = 0
for j in range(5):
avg += self.arr['baseArr'][i]
avg /= 5
tmp = (self.arr['baseArr'][i] + self.arr['baseArr'][i + 1]) / 2
newobj.arr['baseArr'].append(self.arr['baseArr'][i])
newobj.arr['baseArr'].append(
tmp) # + avg - self.arr['baseArr'][i+1])
newobj.arr['baseArr'].append(
self.arr['baseArr'][len(self.arr['baseArr']) - 1])
tmp1 = self.arr['baseArr'][len(self.arr['baseArr']) -
3] - self.arr['baseArr'][
len(self.arr['baseArr']) - 2]
tmp2 = self.arr['baseArr'][len(self.arr['baseArr']) -
2] - self.arr['baseArr'][
len(self.arr['baseArr']) - 1]
tmp = tmp1 - tmp2
newobj.arr['baseArr'].append(
self.arr['baseArr'][len(self.arr['baseArr']) - 1] + tmp)
newobj.N = self.N * 2
newobj.step = self.step / 2
newobj.title = 'Inerpolated'
newobj.parent = self
self.child = newobj
def executePCA(self):
#Execute the PCA Analys on the current Data Object
if (self.dataObject != None):
dboxPCA = dialogs.pcaDialogBox(self.root,
self.dataObject.get_headers())
headers = dboxPCA.getVal(
) #Get the headers for the PCA Data Object
if (len(headers) > 0):
curTime = datetime.datetime.now()
nameNum = curTime.strftime(
"%B %d, %Y") #Creates number based on cur time
analysisObj = analysis.Analysis()
pcaName = "Analysis" + str(curTime.minute) + str(
curTime.second) #Creates the time
if len(dboxPCA.name) > 0:
pcaName = dboxPCA.name
pcaDataObj = analysisObj.pca(self.dataObject,
headers,
normalize=dboxPCA.getNormalize())
pcaDataObj.update_name(pcaName)
self.pcaDataObjects[pcaName] = pcaDataObj
self.updatePCAlistBox()
def trainClassifier(self):
#Execute the PCA Analys on the current Data Object
dboxC = dialogs.trainClassiferDialog(self.root)
if (dboxC.enoughParams == True):
curTime = datetime.datetime.now()
nameNum = curTime.strftime(
"%B %d, %Y") #Creates number based on cur time
analysisObj = analysis.Analysis()
className = "Classifier" + str(curTime.minute) + str(
curTime.second) #Creates the time
trainFile = dboxC.getVals()[0]
testFile = dboxC.getVals()[1]
classType = dboxC.getVals()[2]
kVal = dboxC.getVals()[3]
print trainFile, testFile, classType, kVal
classifier = classify.buildClassifier(trainFile,
testFile,
classType=classType,
K=kVal)
classifier.updateName(className)
self.classifierObjects[className] = classifier
self.updateClassifierListBox()
def __init__(self):
# 创建分析对象
self.ana = analysis.Analysis()
# 创建数据库对象
self.dbm = self.ana.dba
# 获取使用日期
self.DATE = str(time.strftime('%Y-%m-%d', time.localtime()))
def build( self, A, categories, K = None ): '''Builds the classifier give the data points in A and the categories''' # figure out how many categories there are and get the mapping (np.unique) unique, mapping = np.unique( np.array(categories.T), return_inverse=True) row_num=unique.shape[0] # for each category i, build the set of exemplars # if K is None # append to exemplars a matrix with all of the rows of A where the category/mapping is i # else # run K-means on the rows of A where the category/mapping is i # append the codebook to the exemplars for i in range(row_num): if K == None: self.exemplars.append(A[(mapping==i),:]) #use all points else: codebook, codes, error = an.Analysis().kmeans_matrix(A[(mapping==i),:], K, whiten=False) #use points after clustering self.exemplars.append(codebook) # store any other necessary information: # of classes, # of features, original labels self.class_num = row_num self.feature_num = A.shape[1] self.labels = unique return
def main(goodsName=[]):
#先爬取评论数据，并且提取数据，再获取情感分析数据
for i in range(5, len(goodsName)):
spider = JDCommentSpider.CommentSpider(goodsName[i])
spider.getSku(60)
spider.getCommentRun(10, 500)
comm = getCommnetInfo.getCommentInfo(goodsName[i])
comm.run()
ana = analysis.Analysis(goodsName[i])
ana.run(2)
ana.getCloudWrod()
#然后把平均价格和平均差评画出来
ana = analysis.Analysis("")
ana.getBarImage()
ana.getSaticfaction()
ana.getSatic()
def run_analysis(user_to_sig, JSim, estJSim, neighbours):
analysis = anal.Analysis(user_to_sig, JSim, estJSim)
MSE = analysis.MSE()
prec, recall, f1 = analysis.PRF()
print('MSE', MSE)
print('recall ', recall)
def createImageDataset(imageName):
analysisObject = analysis.Analysis(parameters)
analysisObject.outputPath = DATASET_FOLDER
imageDataset = analysisObject.buildDataSetForStimulus(imageName,recordings,imageName,GAZE_DATA)
analysisObject.getGridSize(imageDataset)
analysisObject.analyseDataSet(imageDataset)
#imageDataset.saveToFile(image)
return imageDataset
def antiShift(self):
if (self.analys == None):
analysis.Analysis(self)
newobj = copy.deepcopy(self)
for i in range(len(newobj.arr['baseArr'])):
newobj.arr['baseArr'][i] -= self.analys.M
newobj.title = "Anti Shift"
newobj.parent = self
self.child = newobj
def logrotate_thread_task(writables, tgen_writable, torctl_writable, docroot,
nickname, done_ev):
next_midnight = None
while not done_ev.wait(1):
# get time
utcnow = datetime.datetime.utcnow()
# setup the next expiration time (midnight tonight)
if next_midnight is None:
next_midnight = datetime.datetime(utcnow.year, utcnow.month,
utcnow.day, 23, 59, 59)
# make sure we are not already past the above time today
if (next_midnight - utcnow).total_seconds() < 0:
next_midnight -= datetime.timedelta(1) # subtract 1 day
# if we are past midnight, launch the rotate task
if (next_midnight - utcnow).total_seconds() < 0:
# handle the general writables we are watching
for w in writables:
w.rotate_file()
# handle tgen and tor writables specially, and do analysis
if tgen_writable is not None or torctl_writable is not None:
try:
# set up the analysis object with our log files
anal = analysis.Analysis(nickname=nickname)
if tgen_writable is not None:
anal.add_tgen_file(tgen_writable.rotate_file())
if torctl_writable is not None:
anal.add_torctl_file(torctl_writable.rotate_file())
# run the analysis, i.e. parse the files
anal.analyze(do_simple=False)
# save the results in onionperf and torperf format in the twistd docroot
anal_filename = "{0:04d}-{1:02d}-{2:02d}.onionperf.analysis.json.xz".format(
next_midnight.year, next_midnight.month,
next_midnight.day)
anal.save(filename=anal_filename,
output_prefix=docroot,
do_compress=True)
anal.export_torperf_version_1_0(
output_prefix=docroot,
datetimestamp=next_midnight,
do_compress=False)
# update the xml index in docroot
generate_docroot_index(docroot)
except Exception as e:
logging.warning(
"Caught and ignored exception in TorPerf log parser: {0}"
.format(repr(e)))
logging.warning("Formatted traceback: {0}".format(
traceback.format_exc()))
# reset our timer
next_midnight = None
def buildLinearRegression(self, headers):
analysisObject = analysis.Analysis()
lrObject = analysisObject.normalize_columns_separately(
headers[0:2], self.dataObject)
# lrObject = self.dataObject.get_columns(lrBox.getVal())
zeros = np.matrix(np.zeros(len(lrObject))).transpose()
ones = np.matrix(np.ones(len(lrObject))).transpose()
lrObject = np.hstack((lrObject, zeros)) #Add the third column as zeros
self.spatial_matrix = np.hstack(
(lrObject, ones)) #Add the fourth column as ones
vtm = self.viewObject.build()
pts = (vtm * self.spatial_matrix.transpose()).transpose()
for i in range(len(self.spatial_matrix)):
x = pts[i, 0]
y = pts[i, 1]
dx = self.dataSize
circle = self.canvas.create_oval(
int(x - dx),
int(y - dx),
int(x + dx),
int(y + dx),
tags=(("dtp" +
str(i)))) # Assigns each data point a unique tag
self.objects.append(circle)
slope, intercept, rvalue, pvalue, stdErr = scipy.stats.linregress(
self.preDataObject.get_columns(headers[0:2]))
dataRange = analysisObject.data_range(headers, self.preDataObject)
self.linregressObj = [slope, intercept, rvalue, pvalue, stdErr]
l1 = ((dataRange[0][0] * slope + intercept) -
dataRange[1][0]) / (dataRange[1][1] - dataRange[1][0])
l2 = ((dataRange[0][1] * slope + intercept) -
dataRange[1][0]) / (dataRange[1][1] - dataRange[1][0])
self.pNM = np.matrix( #axes , mainly stays fixed
[[0, 1], [l1, l2], [0, 0], [1, 1]])
pixelCoords = vtm * self.pNM
x0 = pixelCoords[0, 0]
y0 = pixelCoords[1, 0]
x1 = pixelCoords[0, 1]
y1 = pixelCoords[1, 1]
lrLine = self.canvas.create_line(x0,
y0,
x1,
y1,
tags="lrLine",
fill="red")
self.lineObjects.append(lrLine)
lrValsBox = dialogs.LinearRegressionVals(self.root, slope, intercept,
rvalue)
def visualizeMeanFixationDurationHeatmap(imageName, imageDataset):
analysisOb = analysis.Analysis(parameters)
analysisOb.dataFiles = DATASET_FOLDER
plotterOb = analysisOb.getPlotter()
pngFile = imageName.replace(".jpg", ".png", -3)
plotterOb.plotDataSet(
imageDataset, 2, 428.0, " ",
VISUALIZER_FOLDER + pngFile + "MeanFixationLengthHeatmap" + ".png",
VISUALIZER_FOLDER + pngFile)
def visualizeFixationFrequencyHeatmap(imageName, imageDataset):
analysisOb = analysis.Analysis(parameters)
analysisOb.dataFiles = DATASET_FOLDER
plotterOb = analysisOb.getPlotter()
pngFile = imageName.replace(".jpg", ".png", -3)
plotterOb.plotDataSet(
imageDataset, 3, 0.165, " ",
VISUALIZER_FOLDER + pngFile + "fixationFrequencyHeatmap" + ".png",
VISUALIZER_FOLDER + pngFile)
def visualizeTimeToFirstFixation(imageName, imageDataset):
analysisOb = analysis.Analysis(parameters)
analysisOb.dataFiles = DATASET_FOLDER
plotterOb = analysisOb.getPlotter()
pngFile = imageName.replace(".jpg", ".png", -3)
plotterOb.plotDataSet(
imageDataset, 4, 0.15, " ",
VISUALIZER_FOLDER + pngFile + "firstFixationTimeHeatmap" + ".png",
VISUALIZER_FOLDER + pngFile)
def createImageDataset(imageName,eyeTrackingRecordings,datasetLabel):
analysisObject = analysis.Analysis(parameters)
analysisObject.outputPath = DATASET_FOLDER
imageDataset = analysisObject.buildDataSetForStimulus(datasetLabel,eyeTrackingRecordings,imageName,GAZE_DATA)
newParameters = {
'gridWidth': 21, # the grid size: set to None for dynamic calcluation
'gridHeight': None, # the grid size: set to None for square shaped boxes
'errorRadius': 5, # error smoothing sigma (pixels)
'groupingRadius': 50, # filtering radius (pixels)
'fixationLengthFilter': 100 # minimum fixation length
}
analysisObject = analysis.Analysis(newParameters)
#analysisObject.outputPath = DATASET_FOLDER
imageDataset = analysisObject.buildDataSetForStimulus(datasetLabel,eyeTrackingRecordings,imageName,GAZE_DATA)
analysisObject.analyseDataSet(imageDataset)
return imageDataset
def visualizeGazePlots(imageName):
analysisOb = analysis.Analysis(parameters)
analysisOb.datasets = DATASET_FOLDER
plotterOb = analysisOb.getPlotter()
for participant in imageDataset.participantList:
pngFile = imageName.replace(".jpg",".png",-3)
plotterOb.plotFixations([participant],VISUALIZER_FOLDER+imageNames[0]+"_"+participant.number+"_FixationsPlot.png",
VISUALIZER_FOLDER+pngFile)
plotterOb.plotPaths(imageDataset,VISUALIZER_FOLDER+"all_participants_gazePlot",VISUALIZER_FOLDER+pngFile)
def __init__(self):
main_logger = logger.Logger()
r_log = log.Log("RSS Fetcher", main_logger)
s_log = log.Log("Tweet Fetcher", main_logger)
a_log = log.Log("Analysis", main_logger)
self.r = rss_fetcher.RssFetcher(r_log)
self.s = stream_reader.StreamReader(s_log)
self.a = analysis.Analysis(a_log)
def analyze():
analy = analysis.Analysis()
with open("model/sample.txt", "r", encoding="utf-8") as sample_file:
sample = sample_file.read().split("\n")
res_set = []
for sentence in sample:
res = analy.once(sentence)
res_set.append(sentence + "~~~" + str(res))
# res_file = open("model/res.txt", "w", encoding="utf-8")
with open("model/res.txt", "w", encoding="utf-8") as res_file:
for r in res_set:
res_file.write(r)
res_file.write("\n")
def main():
# Replace the directory below with the top-level directory of the data sets
dir = 'E:/ICFO - AFIB/Raid3 (Data)/Multiplexing STORM ALL Data [#####]/STORM DNA-PAINT 2 Color/170405_DNA-paint_MSTORM_Comparison -100ms and 16ms/Overllaped 561nm plus 647nm/'
dir_exts = os.listdir(dir)
# print(dir_exts)
chunk_size = 50000
for dir_ext in dir_exts:
names = os.listdir(dir + dir_ext)
for name in names:
if name[-4:] == '.dax':
daxIn = DAX.DAX()
daxIn.open(dir + dir_ext + '/' + name)
_num_frames = daxIn.numFrames
_num_lasers = len(daxIn.laserWavelengths[::-1])
_mod_freqs = daxIn.modFreqs
_frame_rate = daxIn.frameRate
daxIn.close()
if _num_frames > 100:
num_chunks = _num_frames//chunk_size
# num_chunks = 10
if np.max(_mod_freqs) > 0.48*_frame_rate and np.max(_mod_freqs) < 0.52*_frame_rate:
nyquist = True
else:
nyquist = False
dft_window_width = calc_window_size(_num_lasers, _mod_freqs, _frame_rate, nyquist)
print(name[:-4]+'_demod_'+str(dft_window_width)+'_frame_window')
daxOut = [DAX.DAX() for i in range(_num_lasers)]
for i in range(_num_lasers):
daxOut[i].create(dir+dir_ext+'/'+name[:-4]+'_demod'+str(i)+'_'+str(dft_window_width)+'_frame_window.dax')
start_frame = 0
for i in range(num_chunks):
data = storm.STORM(start = start_frame, frames = chunk_size)
data.parameters['dir'] = dir + dir_ext + '/'
data.parameters['name'] = name
load_success = data.load()
storm_analysis = analysis.Analysis(data, [chunk_size,dft_window_width])
start_time = timer()
analysis_return = storm_analysis.calc_dft()
dt = timer() - start_time
print('Analysis finished in %f s' % dt)
for j in range(_num_lasers):
for k in range(storm_analysis.parameters['d_number']):
daxOut[j].write(analysis_return[j][k])
print('Chunk '+str(i+1)+' of '+str(num_chunks)+' completed.')
storm_analysis.close()
data.close()
start_frame += chunk_size
for i in range(_num_lasers):
daxOut[i].close()
def handleClustering(self):
#Handles clustering for data application
if (self.dataObject != None):
cBox = dialogs.kmeansDialogBox(self.root,
self.dataObject.get_headers())
if (cBox.cancelPress == False):
if (cBox.getVal != None):
if (cBox.getManhattan() == True):
analysisObj = analysis.Analysis()
self.kVal = cBox.name
headers = cBox.getVal()
d = self.dataObject.get_data(
headers, self.dataObject.get_num_rows())
codebook, codes, error = analysisObj.kmeans(
d, headers, self.kVal, manhattan=True)
self.dataObject.add_column(
np.matrix(codes).T, 'codes', 'code')
self.buildPoints()
else:
analysisObj = analysis.Analysis()
self.kVal = cBox.name
headers = cBox.getVal()
d = self.dataObject.get_data(
headers, self.dataObject.get_num_rows())
codebook, codes, error = analysisObj.kmeans_numpy(
d, headers, self.kVal)
self.dataObject.add_column(
np.matrix(codes).T, 'codes', 'code')
self.buildPoints()
else:
print "Please Load a Data Set"
def visualizeHeatmaps(imageName,imageDataset,group):
analysisOb = analysis.Analysis(parameters)
analysisOb.datasets = DATASET_FOLDER
plotterOb = analysisOb.getPlotter()
pngFile = imageName.lower().replace(".jpg",".png",-3)
resultFolder = pngFile.strip(".png") + "\\" + "interestInArtAnalysis" + "\\"
if not os.path.isdir(VISUALIZER_FOLDER+resultFolder):
os.mkdir(VISUALIZER_FOLDER+resultFolder)
plotterOb.plotDataSet(imageDataset,0,1.7," ",
VISUALIZER_FOLDER+resultFolder+"_fixationCountsHeatmap_"+group +".png",VISUALIZER_FOLDER+pngFile)
plotterOb.plotDataSet(imageDataset,2,428.0," ",
VISUALIZER_FOLDER+resultFolder+"_MeanFixationLengthHeatmap_"+group +".png",VISUALIZER_FOLDER+pngFile)
plotterOb.plotDataSet(imageDataset,4,0.15," ",
VISUALIZER_FOLDER+resultFolder+"_firstFixationTimeHeatmap_"+group +".png",VISUALIZER_FOLDER+pngFile)
def visualizeRandomGazeplots(imageName, imageDataset):
analysisOb = analysis.Analysis(parameters)
analysisOb.dataFiles = DATASET_FOLDER
plotterOb = analysisOb.getPlotter()
pngFile = imageName.replace(".jpg", ".png", -3)
sampleSize = len(dataset.participantList)
randomArrayPick = [rnd.randint(0, 66) for i in range(11)]
for element in randomArrayPick:
participant = dataset.participantList.__getitem__(element)
num = participant.number
plotterOb.plotParticipantPaths(imageDataset, num,
VISUALIZER_FOLDER + num + "_gazeplot",
VISUALIZER_FOLDER + pngFile)
def STAT():
try:
os.mkdir("../stat")
except:
pass
print "********************"
print "Initiating statistics..."
# Source directory name
src_dir = "../final/General.txt"
# The path of the result report
res_dir = "../stat/result_final2.txt"
# Create an instance of the 'Analysis' class
s = analysis.Analysis(src_dir, res_dir)
s.run()
def main(data,
JSim,
minhash_type,
num_hashes,
eps=None,
delta=None,
l=None,
b=None,
alpha=None,
num_buckets=None):
user_to_sig, est_JSim = hashing(data, num_hashes, eps, minhash_type, l, b,
alpha, delta, num_buckets)
analysis = anal.Analysis(user_to_sig, JSim, est_JSim)
return analysis
def visualizeHeatmaps(imageName):
analysisOb = analysis.Analysis(parameters)
analysisOb.datasets = DATASET_FOLDER
plotterOb = analysisOb.getPlotter()
pngFile = imageName.replace(".jpg",".png",-3)
# 0 => plot fixation counts
# 1 => fixation duration
# 2 => mean fixation length
# 3 => fixation frequency
# 4 => time to first fixation (currently only gives result for first file in list)
plotterOb.plotDataSet(imageDataset,0,1.4,"Fixation counts heatmap for " +pngFile,VISUALIZER_FOLDER+pngFile+"fixationCountsHeatmap"+".png",VISUALIZER_FOLDER+pngFile)
plotterOb.plotDataSet(imageDataset,1,0.4,"Fixation duration heatmap for " +pngFile,VISUALIZER_FOLDER+pngFile+"fixationDurationHeatmap"+".png",VISUALIZER_FOLDER+pngFile)
plotterOb.plotDataSet(imageDataset,2,0.4,"Mean fixation duration heatmap for " +pngFile,VISUALIZER_FOLDER+pngFile+"MeanFixationLengthHeatmap"+".png",VISUALIZER_FOLDER+pngFile)
plotterOb.plotDataSet(imageDataset,3,0.15,"Fixation frequency heatmap for " +pngFile,VISUALIZER_FOLDER+pngFile+"fixationFrequencyHeatmap"+".png",VISUALIZER_FOLDER+pngFile)
plotterOb.plotDataSet(imageDataset,4,0.15,"Time to first fixation heatmap for"+pngFile,VISUALIZER_FOLDER+pngFile+"firstFixationTimeHeatmap"+".png",VISUALIZER_FOLDER+pngFile)
def viewReports(request, owner, analysisName):
theAnalysis = Analysis.objects.get(name=analysisName)
thisAnalysis = analysis.Analysis(json.loads(theAnalysis.json))
studyList = []
for studyName in thisAnalysis.studyNames:
studyData = json.loads(theAnalysis.study_set.get(name=studyName).json)
studyData['name'] = studyName
studyData['analysisName'] = thisAnalysis.name
moduleRef = getattr(studies, studyData['studyType'])
classRef = getattr(moduleRef, studyData['studyType'].capitalize())
studyList.append(classRef(studyData))
reportList = thisAnalysis.generateReportHtml(studyList)
return TemplateResponse(request, "viewReports.html", {
"analysisName": analysisName,
"reportList": reportList,
"public": True
})
