def handlePCAClustering(self): print self.pcaDataObjects print self.PCAlistbox.curselection() if (bool(self.pcaDataObjects) == True): curSelection = self.PCAlistbox.curselection( ) #List of items to be deleted if (curSelection != None): cBox = dialogs.kmeansDialogBox(self.root, self.dataObject.get_headers()) if (cBox.cancelPress == False): if (cBox.getVal != None): selectVal = self.PCAlistbox.curselection()[ 0] #index value of selected value selectedPCA = self.PCAlistbox.get(selectVal) dataObject = self.pcaDataObjects.get(selectedPCA) if (cBox.getManhattan() == True): analysisObj = analysis.Analysis() self.kVal = cBox.name headers = cBox.getVal() d = dataObject.get_data(headers, dataObject.get_num_rows()) codebook, codes, error = analysisObj.kmeans( d, headers, self.kVal, manhattan=True) dataObject.add_column( np.matrix(codes).T, 'codes', 'code') self.buildPCA(dataObject) else: analysisObj = analysis.Analysis() self.kVal = cBox.name headers = cBox.getVal() d = dataObject.get_data(headers, dataObject.get_num_rows()) codebook, codes, error = analysisObj.kmeans_numpy( d, headers, self.kVal) dataObject.add_column( np.matrix(codes).T, 'codes', 'code') self.buildPCA(dataObject) else: print "Please select a PCA analysis to Run K mean on" else: print "Please perform PCA Analysis first"
def visualizeArbitraryAOI(imageName): analysisOb = analysis.Analysis(parameters) analysisOb.datasets = DATASET_FOLDER plotterOb = analysisOb.getPlotter() pngFile = imageName.replace(".jpg",".png",-3) plotterOb.plotArbitraryAOI(VISUALIZER_FOLDER+"_ArbitraryAOI",VISUALIZER_FOLDER+pngFile)
def visualizeParticipantsGazePlots(imageName,ParticipantNumber): analysisOb = analysis.Analysis(parameters) analysisOb.datasets = DATASET_FOLDER plotterOb = analysisOb.getPlotter() pngFile = imageName.replace(".jpg",".png",-3) plotterOb.plotParticipantPaths(imageDataset,ParticipantNumber,VISUALIZER_FOLDER+ParticipantNumber+"_gazeplot",VISUALIZER_FOLDER+pngFile)
def visualizeHeatmapsByGender(imageName): analysisOb = analysis.Analysis(parameters) analysisOb.datasets = DATASET_FOLDER plotterOb = analysisOb.getPlotter() pngFile = imageName.replace(".jpg",".png",-3)
def inerpolate(self): if (self.analys == None): analysis.Analysis(self) newobj = copy.deepcopy(self) gLen = len(self.arr['baseArr']) newobj.arr['baseArr'] = [] for i in range(gLen - 1): avg = 0 for j in range(5): avg += self.arr['baseArr'][i] avg /= 5 tmp = (self.arr['baseArr'][i] + self.arr['baseArr'][i + 1]) / 2 newobj.arr['baseArr'].append(self.arr['baseArr'][i]) newobj.arr['baseArr'].append( tmp) # + avg - self.arr['baseArr'][i+1]) newobj.arr['baseArr'].append( self.arr['baseArr'][len(self.arr['baseArr']) - 1]) tmp1 = self.arr['baseArr'][len(self.arr['baseArr']) - 3] - self.arr['baseArr'][ len(self.arr['baseArr']) - 2] tmp2 = self.arr['baseArr'][len(self.arr['baseArr']) - 2] - self.arr['baseArr'][ len(self.arr['baseArr']) - 1] tmp = tmp1 - tmp2 newobj.arr['baseArr'].append( self.arr['baseArr'][len(self.arr['baseArr']) - 1] + tmp) newobj.N = self.N * 2 newobj.step = self.step / 2 newobj.title = 'Inerpolated' newobj.parent = self self.child = newobj
def executePCA(self): #Execute the PCA Analys on the current Data Object if (self.dataObject != None): dboxPCA = dialogs.pcaDialogBox(self.root, self.dataObject.get_headers()) headers = dboxPCA.getVal( ) #Get the headers for the PCA Data Object if (len(headers) > 0): curTime = datetime.datetime.now() nameNum = curTime.strftime( "%B %d, %Y") #Creates number based on cur time analysisObj = analysis.Analysis() pcaName = "Analysis" + str(curTime.minute) + str( curTime.second) #Creates the time if len(dboxPCA.name) > 0: pcaName = dboxPCA.name pcaDataObj = analysisObj.pca(self.dataObject, headers, normalize=dboxPCA.getNormalize()) pcaDataObj.update_name(pcaName) self.pcaDataObjects[pcaName] = pcaDataObj self.updatePCAlistBox()
def trainClassifier(self): #Execute the PCA Analys on the current Data Object dboxC = dialogs.trainClassiferDialog(self.root) if (dboxC.enoughParams == True): curTime = datetime.datetime.now() nameNum = curTime.strftime( "%B %d, %Y") #Creates number based on cur time analysisObj = analysis.Analysis() className = "Classifier" + str(curTime.minute) + str( curTime.second) #Creates the time trainFile = dboxC.getVals()[0] testFile = dboxC.getVals()[1] classType = dboxC.getVals()[2] kVal = dboxC.getVals()[3] print trainFile, testFile, classType, kVal classifier = classify.buildClassifier(trainFile, testFile, classType=classType, K=kVal) classifier.updateName(className) self.classifierObjects[className] = classifier self.updateClassifierListBox()
def __init__(self): # 创建分析对象 self.ana = analysis.Analysis() # 创建数据库对象 self.dbm = self.ana.dba # 获取使用日期 self.DATE = str(time.strftime('%Y-%m-%d', time.localtime()))
def build( self, A, categories, K = None ): '''Builds the classifier give the data points in A and the categories''' # figure out how many categories there are and get the mapping (np.unique) unique, mapping = np.unique( np.array(categories.T), return_inverse=True) row_num=unique.shape[0] # for each category i, build the set of exemplars # if K is None # append to exemplars a matrix with all of the rows of A where the category/mapping is i # else # run K-means on the rows of A where the category/mapping is i # append the codebook to the exemplars for i in range(row_num): if K == None: self.exemplars.append(A[(mapping==i),:]) #use all points else: codebook, codes, error = an.Analysis().kmeans_matrix(A[(mapping==i),:], K, whiten=False) #use points after clustering self.exemplars.append(codebook) # store any other necessary information: # of classes, # of features, original labels self.class_num = row_num self.feature_num = A.shape[1] self.labels = unique return
def main(goodsName=[]): #先爬取评论数据,并且提取数据,再获取情感分析数据 for i in range(5, len(goodsName)): spider = JDCommentSpider.CommentSpider(goodsName[i]) spider.getSku(60) spider.getCommentRun(10, 500) comm = getCommnetInfo.getCommentInfo(goodsName[i]) comm.run() ana = analysis.Analysis(goodsName[i]) ana.run(2) ana.getCloudWrod() #然后把平均价格和平均差评画出来 ana = analysis.Analysis("") ana.getBarImage() ana.getSaticfaction() ana.getSatic()
def run_analysis(user_to_sig, JSim, estJSim, neighbours): analysis = anal.Analysis(user_to_sig, JSim, estJSim) MSE = analysis.MSE() prec, recall, f1 = analysis.PRF() print('MSE', MSE) print('recall ', recall)
def createImageDataset(imageName): analysisObject = analysis.Analysis(parameters) analysisObject.outputPath = DATASET_FOLDER imageDataset = analysisObject.buildDataSetForStimulus(imageName,recordings,imageName,GAZE_DATA) analysisObject.getGridSize(imageDataset) analysisObject.analyseDataSet(imageDataset) #imageDataset.saveToFile(image) return imageDataset
def antiShift(self): if (self.analys == None): analysis.Analysis(self) newobj = copy.deepcopy(self) for i in range(len(newobj.arr['baseArr'])): newobj.arr['baseArr'][i] -= self.analys.M newobj.title = "Anti Shift" newobj.parent = self self.child = newobj
def logrotate_thread_task(writables, tgen_writable, torctl_writable, docroot, nickname, done_ev): next_midnight = None while not done_ev.wait(1): # get time utcnow = datetime.datetime.utcnow() # setup the next expiration time (midnight tonight) if next_midnight is None: next_midnight = datetime.datetime(utcnow.year, utcnow.month, utcnow.day, 23, 59, 59) # make sure we are not already past the above time today if (next_midnight - utcnow).total_seconds() < 0: next_midnight -= datetime.timedelta(1) # subtract 1 day # if we are past midnight, launch the rotate task if (next_midnight - utcnow).total_seconds() < 0: # handle the general writables we are watching for w in writables: w.rotate_file() # handle tgen and tor writables specially, and do analysis if tgen_writable is not None or torctl_writable is not None: try: # set up the analysis object with our log files anal = analysis.Analysis(nickname=nickname) if tgen_writable is not None: anal.add_tgen_file(tgen_writable.rotate_file()) if torctl_writable is not None: anal.add_torctl_file(torctl_writable.rotate_file()) # run the analysis, i.e. parse the files anal.analyze(do_simple=False) # save the results in onionperf and torperf format in the twistd docroot anal_filename = "{0:04d}-{1:02d}-{2:02d}.onionperf.analysis.json.xz".format( next_midnight.year, next_midnight.month, next_midnight.day) anal.save(filename=anal_filename, output_prefix=docroot, do_compress=True) anal.export_torperf_version_1_0( output_prefix=docroot, datetimestamp=next_midnight, do_compress=False) # update the xml index in docroot generate_docroot_index(docroot) except Exception as e: logging.warning( "Caught and ignored exception in TorPerf log parser: {0}" .format(repr(e))) logging.warning("Formatted traceback: {0}".format( traceback.format_exc())) # reset our timer next_midnight = None
def buildLinearRegression(self, headers): analysisObject = analysis.Analysis() lrObject = analysisObject.normalize_columns_separately( headers[0:2], self.dataObject) # lrObject = self.dataObject.get_columns(lrBox.getVal()) zeros = np.matrix(np.zeros(len(lrObject))).transpose() ones = np.matrix(np.ones(len(lrObject))).transpose() lrObject = np.hstack((lrObject, zeros)) #Add the third column as zeros self.spatial_matrix = np.hstack( (lrObject, ones)) #Add the fourth column as ones vtm = self.viewObject.build() pts = (vtm * self.spatial_matrix.transpose()).transpose() for i in range(len(self.spatial_matrix)): x = pts[i, 0] y = pts[i, 1] dx = self.dataSize circle = self.canvas.create_oval( int(x - dx), int(y - dx), int(x + dx), int(y + dx), tags=(("dtp" + str(i)))) # Assigns each data point a unique tag self.objects.append(circle) slope, intercept, rvalue, pvalue, stdErr = scipy.stats.linregress( self.preDataObject.get_columns(headers[0:2])) dataRange = analysisObject.data_range(headers, self.preDataObject) self.linregressObj = [slope, intercept, rvalue, pvalue, stdErr] l1 = ((dataRange[0][0] * slope + intercept) - dataRange[1][0]) / (dataRange[1][1] - dataRange[1][0]) l2 = ((dataRange[0][1] * slope + intercept) - dataRange[1][0]) / (dataRange[1][1] - dataRange[1][0]) self.pNM = np.matrix( #axes , mainly stays fixed [[0, 1], [l1, l2], [0, 0], [1, 1]]) pixelCoords = vtm * self.pNM x0 = pixelCoords[0, 0] y0 = pixelCoords[1, 0] x1 = pixelCoords[0, 1] y1 = pixelCoords[1, 1] lrLine = self.canvas.create_line(x0, y0, x1, y1, tags="lrLine", fill="red") self.lineObjects.append(lrLine) lrValsBox = dialogs.LinearRegressionVals(self.root, slope, intercept, rvalue)
def visualizeMeanFixationDurationHeatmap(imageName, imageDataset): analysisOb = analysis.Analysis(parameters) analysisOb.dataFiles = DATASET_FOLDER plotterOb = analysisOb.getPlotter() pngFile = imageName.replace(".jpg", ".png", -3) plotterOb.plotDataSet( imageDataset, 2, 428.0, " ", VISUALIZER_FOLDER + pngFile + "MeanFixationLengthHeatmap" + ".png", VISUALIZER_FOLDER + pngFile)
def visualizeFixationFrequencyHeatmap(imageName, imageDataset): analysisOb = analysis.Analysis(parameters) analysisOb.dataFiles = DATASET_FOLDER plotterOb = analysisOb.getPlotter() pngFile = imageName.replace(".jpg", ".png", -3) plotterOb.plotDataSet( imageDataset, 3, 0.165, " ", VISUALIZER_FOLDER + pngFile + "fixationFrequencyHeatmap" + ".png", VISUALIZER_FOLDER + pngFile)
def visualizeTimeToFirstFixation(imageName, imageDataset): analysisOb = analysis.Analysis(parameters) analysisOb.dataFiles = DATASET_FOLDER plotterOb = analysisOb.getPlotter() pngFile = imageName.replace(".jpg", ".png", -3) plotterOb.plotDataSet( imageDataset, 4, 0.15, " ", VISUALIZER_FOLDER + pngFile + "firstFixationTimeHeatmap" + ".png", VISUALIZER_FOLDER + pngFile)
def createImageDataset(imageName,eyeTrackingRecordings,datasetLabel): analysisObject = analysis.Analysis(parameters) analysisObject.outputPath = DATASET_FOLDER imageDataset = analysisObject.buildDataSetForStimulus(datasetLabel,eyeTrackingRecordings,imageName,GAZE_DATA) newParameters = { 'gridWidth': 21, # the grid size: set to None for dynamic calcluation 'gridHeight': None, # the grid size: set to None for square shaped boxes 'errorRadius': 5, # error smoothing sigma (pixels) 'groupingRadius': 50, # filtering radius (pixels) 'fixationLengthFilter': 100 # minimum fixation length } analysisObject = analysis.Analysis(newParameters) #analysisObject.outputPath = DATASET_FOLDER imageDataset = analysisObject.buildDataSetForStimulus(datasetLabel,eyeTrackingRecordings,imageName,GAZE_DATA) analysisObject.analyseDataSet(imageDataset) return imageDataset
def visualizeGazePlots(imageName): analysisOb = analysis.Analysis(parameters) analysisOb.datasets = DATASET_FOLDER plotterOb = analysisOb.getPlotter() for participant in imageDataset.participantList: pngFile = imageName.replace(".jpg",".png",-3) plotterOb.plotFixations([participant],VISUALIZER_FOLDER+imageNames[0]+"_"+participant.number+"_FixationsPlot.png", VISUALIZER_FOLDER+pngFile) plotterOb.plotPaths(imageDataset,VISUALIZER_FOLDER+"all_participants_gazePlot",VISUALIZER_FOLDER+pngFile)
def __init__(self): main_logger = logger.Logger() r_log = log.Log("RSS Fetcher", main_logger) s_log = log.Log("Tweet Fetcher", main_logger) a_log = log.Log("Analysis", main_logger) self.r = rss_fetcher.RssFetcher(r_log) self.s = stream_reader.StreamReader(s_log) self.a = analysis.Analysis(a_log)
def analyze(): analy = analysis.Analysis() with open("model/sample.txt", "r", encoding="utf-8") as sample_file: sample = sample_file.read().split("\n") res_set = [] for sentence in sample: res = analy.once(sentence) res_set.append(sentence + "~~~" + str(res)) # res_file = open("model/res.txt", "w", encoding="utf-8") with open("model/res.txt", "w", encoding="utf-8") as res_file: for r in res_set: res_file.write(r) res_file.write("\n")
def main(): # Replace the directory below with the top-level directory of the data sets dir = 'E:/ICFO - AFIB/Raid3 (Data)/Multiplexing STORM ALL Data [#####]/STORM DNA-PAINT 2 Color/170405_DNA-paint_MSTORM_Comparison -100ms and 16ms/Overllaped 561nm plus 647nm/' dir_exts = os.listdir(dir) # print(dir_exts) chunk_size = 50000 for dir_ext in dir_exts: names = os.listdir(dir + dir_ext) for name in names: if name[-4:] == '.dax': daxIn = DAX.DAX() daxIn.open(dir + dir_ext + '/' + name) _num_frames = daxIn.numFrames _num_lasers = len(daxIn.laserWavelengths[::-1]) _mod_freqs = daxIn.modFreqs _frame_rate = daxIn.frameRate daxIn.close() if _num_frames > 100: num_chunks = _num_frames//chunk_size # num_chunks = 10 if np.max(_mod_freqs) > 0.48*_frame_rate and np.max(_mod_freqs) < 0.52*_frame_rate: nyquist = True else: nyquist = False dft_window_width = calc_window_size(_num_lasers, _mod_freqs, _frame_rate, nyquist) print(name[:-4]+'_demod_'+str(dft_window_width)+'_frame_window') daxOut = [DAX.DAX() for i in range(_num_lasers)] for i in range(_num_lasers): daxOut[i].create(dir+dir_ext+'/'+name[:-4]+'_demod'+str(i)+'_'+str(dft_window_width)+'_frame_window.dax') start_frame = 0 for i in range(num_chunks): data = storm.STORM(start = start_frame, frames = chunk_size) data.parameters['dir'] = dir + dir_ext + '/' data.parameters['name'] = name load_success = data.load() storm_analysis = analysis.Analysis(data, [chunk_size,dft_window_width]) start_time = timer() analysis_return = storm_analysis.calc_dft() dt = timer() - start_time print('Analysis finished in %f s' % dt) for j in range(_num_lasers): for k in range(storm_analysis.parameters['d_number']): daxOut[j].write(analysis_return[j][k]) print('Chunk '+str(i+1)+' of '+str(num_chunks)+' completed.') storm_analysis.close() data.close() start_frame += chunk_size for i in range(_num_lasers): daxOut[i].close()
def handleClustering(self): #Handles clustering for data application if (self.dataObject != None): cBox = dialogs.kmeansDialogBox(self.root, self.dataObject.get_headers()) if (cBox.cancelPress == False): if (cBox.getVal != None): if (cBox.getManhattan() == True): analysisObj = analysis.Analysis() self.kVal = cBox.name headers = cBox.getVal() d = self.dataObject.get_data( headers, self.dataObject.get_num_rows()) codebook, codes, error = analysisObj.kmeans( d, headers, self.kVal, manhattan=True) self.dataObject.add_column( np.matrix(codes).T, 'codes', 'code') self.buildPoints() else: analysisObj = analysis.Analysis() self.kVal = cBox.name headers = cBox.getVal() d = self.dataObject.get_data( headers, self.dataObject.get_num_rows()) codebook, codes, error = analysisObj.kmeans_numpy( d, headers, self.kVal) self.dataObject.add_column( np.matrix(codes).T, 'codes', 'code') self.buildPoints() else: print "Please Load a Data Set"
def visualizeHeatmaps(imageName,imageDataset,group): analysisOb = analysis.Analysis(parameters) analysisOb.datasets = DATASET_FOLDER plotterOb = analysisOb.getPlotter() pngFile = imageName.lower().replace(".jpg",".png",-3) resultFolder = pngFile.strip(".png") + "\\" + "interestInArtAnalysis" + "\\" if not os.path.isdir(VISUALIZER_FOLDER+resultFolder): os.mkdir(VISUALIZER_FOLDER+resultFolder) plotterOb.plotDataSet(imageDataset,0,1.7," ", VISUALIZER_FOLDER+resultFolder+"_fixationCountsHeatmap_"+group +".png",VISUALIZER_FOLDER+pngFile) plotterOb.plotDataSet(imageDataset,2,428.0," ", VISUALIZER_FOLDER+resultFolder+"_MeanFixationLengthHeatmap_"+group +".png",VISUALIZER_FOLDER+pngFile) plotterOb.plotDataSet(imageDataset,4,0.15," ", VISUALIZER_FOLDER+resultFolder+"_firstFixationTimeHeatmap_"+group +".png",VISUALIZER_FOLDER+pngFile)
def visualizeRandomGazeplots(imageName, imageDataset): analysisOb = analysis.Analysis(parameters) analysisOb.dataFiles = DATASET_FOLDER plotterOb = analysisOb.getPlotter() pngFile = imageName.replace(".jpg", ".png", -3) sampleSize = len(dataset.participantList) randomArrayPick = [rnd.randint(0, 66) for i in range(11)] for element in randomArrayPick: participant = dataset.participantList.__getitem__(element) num = participant.number plotterOb.plotParticipantPaths(imageDataset, num, VISUALIZER_FOLDER + num + "_gazeplot", VISUALIZER_FOLDER + pngFile)
def STAT(): try: os.mkdir("../stat") except: pass print "********************" print "Initiating statistics..." # Source directory name src_dir = "../final/General.txt" # The path of the result report res_dir = "../stat/result_final2.txt" # Create an instance of the 'Analysis' class s = analysis.Analysis(src_dir, res_dir) s.run()
def main(data, JSim, minhash_type, num_hashes, eps=None, delta=None, l=None, b=None, alpha=None, num_buckets=None): user_to_sig, est_JSim = hashing(data, num_hashes, eps, minhash_type, l, b, alpha, delta, num_buckets) analysis = anal.Analysis(user_to_sig, JSim, est_JSim) return analysis
def visualizeHeatmaps(imageName): analysisOb = analysis.Analysis(parameters) analysisOb.datasets = DATASET_FOLDER plotterOb = analysisOb.getPlotter() pngFile = imageName.replace(".jpg",".png",-3) # 0 => plot fixation counts # 1 => fixation duration # 2 => mean fixation length # 3 => fixation frequency # 4 => time to first fixation (currently only gives result for first file in list) plotterOb.plotDataSet(imageDataset,0,1.4,"Fixation counts heatmap for " +pngFile,VISUALIZER_FOLDER+pngFile+"fixationCountsHeatmap"+".png",VISUALIZER_FOLDER+pngFile) plotterOb.plotDataSet(imageDataset,1,0.4,"Fixation duration heatmap for " +pngFile,VISUALIZER_FOLDER+pngFile+"fixationDurationHeatmap"+".png",VISUALIZER_FOLDER+pngFile) plotterOb.plotDataSet(imageDataset,2,0.4,"Mean fixation duration heatmap for " +pngFile,VISUALIZER_FOLDER+pngFile+"MeanFixationLengthHeatmap"+".png",VISUALIZER_FOLDER+pngFile) plotterOb.plotDataSet(imageDataset,3,0.15,"Fixation frequency heatmap for " +pngFile,VISUALIZER_FOLDER+pngFile+"fixationFrequencyHeatmap"+".png",VISUALIZER_FOLDER+pngFile) plotterOb.plotDataSet(imageDataset,4,0.15,"Time to first fixation heatmap for"+pngFile,VISUALIZER_FOLDER+pngFile+"firstFixationTimeHeatmap"+".png",VISUALIZER_FOLDER+pngFile)
def viewReports(request, owner, analysisName): theAnalysis = Analysis.objects.get(name=analysisName) thisAnalysis = analysis.Analysis(json.loads(theAnalysis.json)) studyList = [] for studyName in thisAnalysis.studyNames: studyData = json.loads(theAnalysis.study_set.get(name=studyName).json) studyData['name'] = studyName studyData['analysisName'] = thisAnalysis.name moduleRef = getattr(studies, studyData['studyType']) classRef = getattr(moduleRef, studyData['studyType'].capitalize()) studyList.append(classRef(studyData)) reportList = thisAnalysis.generateReportHtml(studyList) return TemplateResponse(request, "viewReports.html", { "analysisName": analysisName, "reportList": reportList, "public": True })