def generateCombinedStatsFile(): for normalData, mrData in zip( FileIO.iterateJsonFromFile(TweetsFile.stats_file), FileIO.iterateJsonFromFile(TweetsFile.mr_stats_file)): normalData['mr_k_means'] = mrData['mr_k_means'] FileIO.writeToFileAsJson(normalData, TweetsFile.combined_stats_file)
def plotQualityWithKMeansAndSSA(): del plotSettings["ssa_mr"] speedStats = dict([(k, {"f1": [], "nmi": [], "purity": []}) for k in plotSettings]) for data in FileIO.iterateJsonFromFile(TweetsFile.stats_file): for k in speedStats: for metric in speedStats["ssa"]: speedStats[k][metric].append(data[k][metric]) for k in speedStats: del speedStats[k]["f1"] speedStats.update(dict([(k, {"f1": [], "nmi": [], "purity": []}) for k in kMeansPlotSettings])) k = "k_means" for data in FileIO.iterateJsonFromFile(TweetsFile.combined_stats_file): for metric in speedStats["k_means"]: speedStats[k][metric].append(data[k][metric]) for k in speedStats: if "f1" in speedStats[k]: del speedStats[k]["f1"] dataForPlot = dict([(k, []) for k in speedStats]) for k in speedStats: for k1 in speedStats[k]: dataForPlot[k] += [np.mean(speedStats[k][k1])] # del dataForPlot['k_means'] print dataForPlot ind, width = np.arange(2), 0.1 rects, i = [], 1 plotSettings.update(kMeansPlotSettings) for k in dataForPlot: rects.append(plt.bar(ind + i * width, dataForPlot[k], width, color=plotSettings[k]["color"])) i += 1 plt.ylabel(getLatexForString("Score")) plt.title(getLatexForString("Clustering quality comparison for Streaming LSH with SSA")) plt.xticks(ind + 2 * width, ("$Purity$", "$NMI$")) plt.legend([r[0] for r in rects], [plotSettings[k]["label"] for k in plotSettings], loc=4) # plt.show() plt.savefig("qualityComparisonAll.pdf")
def generate_data_for_significant_nei_utm_ids(): output_file = GeneralMethods.get_method_id()+'.json' so_hashtags, mf_utm_id_to_valid_nei_utm_ids = set(), {} for utm_object in \ FileIO.iterateJsonFromFile(f_hashtags_by_utm_id, True): for hashtag, count in utm_object['mf_hashtag_to_count'].iteritems(): if hashtag!='total_num_of_occurrences': so_hashtags.add(hashtag) mf_utm_id_to_valid_nei_utm_ids[utm_object['utm_id']] =\ utm_object['mf_nei_utm_id_to_common_h_count'].keys() hashtags = sorted(list(so_hashtags)) mf_utm_id_to_vector = {} for utm_object in FileIO.iterateJsonFromFile(f_hashtags_by_utm_id, True): # print i, utm_object['utm_id'] utm_id_vector = map(lambda hashtag: utm_object['mf_hashtag_to_count'].get(hashtag, 0.0), hashtags) mf_utm_id_to_vector[utm_object['utm_id']] = robjects.FloatVector(utm_id_vector) for i, (utm_id, vector) in enumerate(mf_utm_id_to_vector.iteritems()): print '%s of %s'%(i+1, len(mf_utm_id_to_vector)) ltuo_utm_id_and_vector = [(utm_id, vector)] for valid_nei_utm_id in mf_utm_id_to_valid_nei_utm_ids[utm_id]: if valid_nei_utm_id in mf_utm_id_to_vector and valid_nei_utm_id!=utm_id: ltuo_utm_id_and_vector.append((valid_nei_utm_id, mf_utm_id_to_vector[valid_nei_utm_id])) od = rlc.OrdDict(sorted(ltuo_utm_id_and_vector, key=itemgetter(0))) df_utm_vectors = robjects.DataFrame(od) df_utm_vectors_json = R_Helper.get_json_for_data_frame(df_utm_vectors) dfm_dict = cjson.decode(df_utm_vectors_json) mf_utm_ids_to_utm_colnames = dict(zip(zip(*ltuo_utm_id_and_vector)[0], df_utm_vectors.colnames)) utm_id_colname = mf_utm_ids_to_utm_colnames[utm_id] dfm_dict['prediction_variable'] = utm_id_colname dfm_dict['predictor_variables'] = filter(lambda colname: colname!=utm_id_colname, df_utm_vectors.colnames) dfm_dict['mf_utm_colnames_to_utm_ids'] = dict(zip(df_utm_vectors.colnames, zip(*ltuo_utm_id_and_vector)[0])) FileIO.writeToFileAsJson(dfm_dict, output_file)
def build(numberOfTimeUnits=24): validLattices = set() for data in FileIO.iterateJsonFromFile(hashtagsLatticeGraphFile%('world','%s_%s'%(2,11))): validLattices.add(data['id']) documents, lattices = [], set() for h in FileIO.iterateJsonFromFile(hashtagsFile%('training_world','%s_%s'%(2,11))): hashtag, document = Hashtag(h), [] if hashtag.isValidObject(): for timeUnit, occs in enumerate(hashtag.getOccrancesEveryTimeWindowIterator(HashtagsClassifier.CLASSIFIER_TIME_UNIT_IN_SECONDS)): occs = filter(lambda t: t[0] in validLattices, occs) occs = sorted(occs, key=itemgetter(0)) if occs: for lattice in zip(*occs)[0]: lattices.add(lattice) document.append([timeUnit, [(k, len(list(i))) for k, i in groupby(occs, key=itemgetter(0))]]) if document: documents.append(document) lattices = sorted(list(lattices)) print len(lattices) documents = [(d, TargetSelectionRegressionClassifier.getPercentageDistributionInLattice(d)) for d in documents] documents = documents[:int(len(documents)*0.80)] for decisionTimeUnit in range(1, numberOfTimeUnits+1): for latticeCount, predictingLattice in enumerate(lattices): print decisionTimeUnit, latticeCount, inputVectors, outputValues = [], [] for rawDocument, processedDocument in documents: documentForTimeUnit = TargetSelectionRegressionClassifier.getPercentageDistributionInLattice(rawDocument[:decisionTimeUnit]) if documentForTimeUnit and processedDocument: vector = [documentForTimeUnit.get(l, 0) for l in lattices] inputVectors.append(vector), outputValues.append(float(processedDocument.get(predictingLattice, 0))) # TargetSelectionRegressionClassifier(decisionTimeUnit=decisionTimeUnit, predictingLattice=predictingLattice).build(zip(inputVectors, outputValues)) TargetSelectionRegressionSVMRBFClassifier(decisionTimeUnit=decisionTimeUnit, predictingLattice=predictingLattice).build(zip(inputVectors, outputValues))
def plotQualityWithKMeansAndSSA(): del plotSettings['ssa_mr'] speedStats = dict([(k, { 'f1': [], 'nmi': [], 'purity': [] }) for k in plotSettings]) for data in FileIO.iterateJsonFromFile(TweetsFile.stats_file): for k in speedStats: for metric in speedStats['ssa']: speedStats[k][metric].append(data[k][metric]) for k in speedStats: del speedStats[k]['f1'] speedStats.update( dict([(k, { 'f1': [], 'nmi': [], 'purity': [] }) for k in kMeansPlotSettings])) k = 'k_means' for data in FileIO.iterateJsonFromFile(TweetsFile.combined_stats_file): for metric in speedStats['k_means']: speedStats[k][metric].append(data[k][metric]) for k in speedStats: if 'f1' in speedStats[k]: del speedStats[k]['f1'] dataForPlot = dict([(k, []) for k in speedStats]) for k in speedStats: for k1 in speedStats[k]: dataForPlot[k] += [np.mean(speedStats[k][k1])] # del dataForPlot['k_means'] print dataForPlot ind, width = np.arange(2), 0.1 rects, i = [], 1 plotSettings.update(kMeansPlotSettings) for k in dataForPlot: rects.append( plt.bar(ind + i * width, dataForPlot[k], width, color=plotSettings[k]['color'])) i += 1 plt.ylabel(getLatexForString('Score')) plt.title( getLatexForString( 'Clustering quality comparison for Streaming LSH with SSA')) plt.xticks(ind + 2 * width, ('$Purity$', '$NMI$')) plt.legend([r[0] for r in rects], [plotSettings[k]['label'] for k in plotSettings], loc=4) # plt.show() plt.savefig('qualityComparisonAll.pdf')
def coverageIndication(): MINUTES = 5 for timeUnit, color, shape in [(1, 'r', 'x'), (3, 'g', 'd'), (6, 'b', 's')]: print timeUnit data = defaultdict(int) for hashtagObject in FileIO.iterateJsonFromFile(hashtagsFile%('training_world','%s_%s'%(2,11))): try: occsDistributionInTimeUnits = getOccurranceDistributionInEpochs(getOccuranesInHighestActiveRegion(hashtagObject), timeUnit=MINUTES*60, fillInGaps=True, occurancesCount=False) occurances = list(zip(*sorted(occsDistributionInTimeUnits.iteritems(), key=itemgetter(0)))[1]) occsInTimeunit = zip(*reduce(lambda aggList, l: aggList+l, occurances[:timeUnit], []))[0] if len(occsInTimeunit)>10: allOccurances = zip(*reduce(lambda aggList, l: aggList+l, occurances, []))[0] timeUnitRadius, allRadius = getRadius(occsInTimeunit), getRadius(allOccurances) data[int(abs(timeUnitRadius-allRadius))/50*50+50]+=1 # data[round(abs(timeUnitRadius-allRadius)/allRadius, 2)]+=1 except IndexError as e: pass for k in data.keys()[:]: if data[k]<3: del data[k] dataX, dataY = zip(*sorted(data.iteritems(), key=itemgetter(0))) plt.loglog(dataX, dataY, lw=2, label=str(timeUnit*MINUTES) + ' minutes', marker=shape) # plt.loglog([1],[1]) # plt.title('Early indication of coverage'), plt.xlabel('Coverage difference (miles)', fontsize=20), plt.ylabel('Number of hashtags', fontsize=20) plt.legend() # plt.show() plt.savefig('../images/coverageIndication.png')
def temporalLocalityTemporalDistanceExample(lattice=NEW_YORK): distances = defaultdict(dict) for latticeObject in FileIO.iterateJsonFromFile(hashtagsLatticeGraphFile%('training_world','%s_%s'%(2,11))): if latticeObject['id']==lattice: latticeHashtagsSet = set(latticeObject['hashtags']) for neighborLattice, neighborHashtags in latticeObject['links'].iteritems(): distances[neighborLattice] = {} neighborHashtags = filterOutNeighborHashtagsOutside1_5IQROfTemporalDistance(latticeObject['hashtags'], neighborHashtags, findLag=False) neighborHashtagsSet = set(neighborHashtags) distances[neighborLattice]['similarity']=len(latticeHashtagsSet.intersection(neighborHashtagsSet))/float(len(latticeHashtagsSet.union(neighborHashtagsSet))) distances[neighborLattice]['temporalDistance']=np.mean([abs(latticeObject['hashtags'][k][0]-neighborHashtags[k][0]) for k in neighborHashtags if k in latticeObject['hashtags']])/(60.*60.) distances[neighborLattice]['geoDistance']=getHaversineDistanceForLids(latticeObject['id'].replace('_', ' '), neighborLattice.replace('_', ' ')) break dataPoints = [] ax = plt.subplot(111) for k, data in distances.iteritems(): dataPoints.append((getLocationFromLid(k.replace('_', ' ')), data['temporalDistance'])) points, colors = zip(*sorted(dataPoints, key=itemgetter(1))) sc = plotPointsOnWorldMap(points, blueMarble=False, bkcolor='#CFCFCF', cmap='RdPu', c=colors, lw = 0, alpha=1.0) plotPointsOnWorldMap([getLocationFromLid(lattice.replace('_', ' '))], blueMarble=False, bkcolor='#CFCFCF', c='#64FF1C', lw = 0) divider = make_axes_locatable(ax) plt.title('Average time difference from New York') cax = divider.append_axes("right", size="5%", pad=0.05) plt.colorbar(sc, cax=cax) # plt.show() plt.savefig('../images/temporalDistanceExample.png')
def plotClusteringSpeed(saveFig=True): plotSettings = { "k_means": {"label": "Iterative k-means", "color": "#FD0006"}, "mr_k_means": {"label": "MR k-means", "color": "#5AF522"}, "streaming_lsh": {"label": "Stream CDA", "color": "#7109AA"}, } dataToPlot = { "k_means": {"x": [], "y": []}, "mr_k_means": {"x": [], "y": []}, "streaming_lsh": {"x": [], "y": []}, } for data in FileIO.iterateJsonFromFile(TweetsFile.combined_stats_file): for k in plotSettings: dataToPlot[k]["x"].append(data[k]["no_of_documents"]) dataToPlot[k]["y"].append(data[k]["iteration_time"]) for k in plotSettings: plt.loglog( dataToPlot[k]["x"], dataToPlot[k]["y"], label=plotSettings[k]["label"], color=plotSettings[k]["color"], lw=2, ) plt.legend(loc=4) if saveFig: plt.xlabel(getLatexForString("\# of documents")) plt.ylabel(getLatexForString("Running time (s)")) plt.title(getLatexForString("Running time comparsion for Streaing LSH with k-Means")) plt.xlim(xmin=800, xmax=100000) plt.xticks([]) # plt.show() if saveFig: plt.savefig("speedComparisonWithKMeans.pdf")
def plotClusteringQuality(): del plotSettings["ssa_mr"] speedStats = dict([(k, {"f1": [], "nmi": [], "purity": []}) for k in plotSettings]) for data in FileIO.iterateJsonFromFile(TweetsFile.stats_file): for k in speedStats: for metric in speedStats["ssa"]: speedStats[k][metric].append(data[k][metric]) dataForPlot = dict([(k, []) for k in plotSettings]) for k, v in speedStats.iteritems(): print k for k1, v1 in v.iteritems(): if type(v1[0]) != type([]): print k1, "(%0.2f %0.2f)" % (np.mean(v1), np.var(v1)) dataForPlot[k] += [np.mean(v1)] else: print k1, ["(%0.2f %0.2f)" % (np.mean(z), np.var(z)) for z in zip(*v1)] dataForPlot[k] += [np.mean(z) for z in zip(*v1)] ind, width = np.arange(5), 0.1 rects, i = [], 0 for k in dataForPlot: rects.append(plt.bar(ind + i * width, dataForPlot[k], width, color=plotSettings[k]["color"])) i += 1 plt.ylabel(getLatexForString("Score")) plt.title(getLatexForString("Clustering quality comparison for Streaming LSH with SSA")) plt.xticks(ind + width, ("$F$", "$Precision$", "$Recall$", "$Purity$", "$NMI$")) plt.legend([r[0] for r in rects], [plotSettings[k]["label"] for k in plotSettings], loc=4) # plt.show() plt.savefig("qualityComparisonWithSSA.pdf")
def plotClusteringSpeed(saveFig=True): dataToPlot = dict([(k, {"x": [], "y": []}) for k in plotSettings]) for data in FileIO.iterateJsonFromFile(TweetsFile.stats_file): for k in plotSettings: dataToPlot[k]["x"].append(data[k]["no_of_documents"]) dataToPlot[k]["y"].append(data[k]["iteration_time"]) for k in plotSettings: plt.loglog( dataToPlot[k]["x"], movingAverage(dataToPlot[k]["y"], 1), label=plotSettings[k]["label"], color=plotSettings[k]["color"], lw=2, ) print dataToPlot["streaming_lsh"]["x"][10] print dataToPlot["streaming_lsh"]["y"][10] plt.legend(loc=4) if saveFig: plt.xlabel(getLatexForString("\# of documents")) plt.ylabel(getLatexForString("Running time (s)")) plt.title(getLatexForString("Running time comparsion for Streaing LSH with SSA")) plt.xlim(xmin=500, xmax=600000) # plt.show() if saveFig: plt.savefig("speedComparisonWithSSA.pdf")
def measureCorrelations(timeRange, outputFolder): ''' ['haversine_distance', 'temporal_distance_in_hours', 0.20147108648121248] ['haversine_distance', 'sharing_probability', -0.19587239643328627] ''' measures = [ (LatticeGraph.typeHaversineDistance, LatticeGraph.typeTemporalDistanceInHours), (LatticeGraph.typeHaversineDistance, LatticeGraph.typeSharingProbability), ] runData = [] for xMeasure, yMeasure in measures: i, xdata, ydata = 1, [], [] for latticeObject in FileIO.iterateJsonFromFile(hashtagsLatticeGraphFile%(outputFolder,'%s_%s'%timeRange)): print i, latticeObject['id']; i+=1 xdata+=zip(*xMeasure['method'](latticeObject)['links'].iteritems())[1] ydata+=zip(*yMeasure['method'](latticeObject)['links'].iteritems())[1] # if i==200: break preasonsCorrelation, _ = stats.pearsonr(xdata, ydata) # plt.scatter(xdata[:5000], ydata[:5000]) # plt.title('Pearson\'s co-efficient %0.3f'%preasonsCorrelation) # plt.xlabel(xMeasure['title']), plt.ylabel(yMeasure['title']) # plt.show() runData.append([xMeasure['id'], yMeasure['id'], preasonsCorrelation]) for i in runData: print i
def plotCDFClustersLagDistribution(self, returnAxisValuesOnly=True): ''' This determines the time after which a cluster can be considered decayed and hence removed. Experts stream [ 0.66002386 0.07035227] 0.1 82 Houston stream [ 0.73800037 0.05890473] 0.1 29 458 (# of time units) Experts stream [ 0.66002386 0.07035227] 0.2 15 71 (# of time units) Houston stream [ 0.73756656 0.05883258] 0.2 3 ''' def calculateInActivityTimeFor(params, probabilityOfInactivity): return int(CurveFit.inverseOfIncreasingExponentialFunction(params, 1 - probabilityOfInactivity)) data = list(FileIO.iterateJsonFromFile(self.hdsClustering.stream_settings['%s_file' % ClusteringParametersEstimation.clusterLagDistributionId]))[-1] total = float(sum(data['lag_between_streams_added_to_cluster'].values())) x = sorted(map(int, data['lag_between_streams_added_to_cluster'].keys())) y = getCumulativeDistribution([data['lag_between_streams_added_to_cluster'][str(i)] / total for i in x]) exponentialCurveParams = CurveFit.getParamsAfterFittingData(x, y, CurveFit.increasingExponentialFunction, [1., 1.]) print self.stream_settings['plot_label'], exponentialCurveParams, calculateInActivityTimeFor(exponentialCurveParams, 0.2) plt.plot(x, y, 'o', label=getLatexForString(self.stream_settings['plot_label']) + getLatexForString(' (%0.2fx^{%0.2f})') % (exponentialCurveParams[0], exponentialCurveParams[1]), color=self.stream_settings['plot_color']) plt.plot(x, CurveFit.getYValues(CurveFit.increasingExponentialFunction, exponentialCurveParams, x), color=self.stream_settings['plot_color'], lw=2) plt.ylabel(r'$P\ (\ lag\ \leq\ TU\ )$'), plt.xlabel(getLatexForString(xlabelTimeUnits)), plt.title(getLatexForString('CDF for clusters lag distribution.')) plt.ylim((0, 1.2)) plt.legend(loc=4) if returnAxisValuesOnly: plt.show()
def probabilisticCoverageModelExample(hashtag, type): MINUTES, timeUnit = 5, 1 print len(CoverageBasedLatticeSelectionModel.lattices) for hashtagObject in FileIO.iterateJsonFromFile('/mnt/chevron/kykamath/data/geo/hashtags/analysis/all_world/2_11/hashtagsWithoutEndingWindow'): if hashtagObject['h']==hashtag: occsDistributionInTimeUnits = getOccurranceDistributionInEpochs(getOccuranesInHighestActiveRegion(hashtagObject), timeUnit=MINUTES*60, fillInGaps=True, occurancesCount=False) occurances = list(zip(*sorted(occsDistributionInTimeUnits.iteritems(), key=itemgetter(0)))[1]) occsInTimeunit = zip(*reduce(lambda aggList, l: aggList+l, occurances[:timeUnit], []))[0] allOccurances = zip(*reduce(lambda aggList, l: aggList+l, occurances, []))[0] if type=='5m': probabilityDistributionForObservedLattices = CoverageBasedLatticeSelectionModel.probabilityDistributionForLattices(occsInTimeunit) else: print getRadius(allOccurances) probabilityDistributionForObservedLattices = CoverageBasedLatticeSelectionModel.probabilityDistributionForLattices(allOccurances) latticeScores = CoverageBasedLatticeSelectionModel.spreadProbability(CoverageBasedLatticeSelectionModel.lattices, probabilityDistributionForObservedLattices) points, colors = zip(*map(lambda t: (getLocationFromLid(t[0].replace('_', ' ')), t[1]), sorted(latticeScores.iteritems(), key=itemgetter(1)))) # print points[0], colors[0] ax = plt.subplot(111) sc = plotPointsOnWorldMap(points, blueMarble=False, bkcolor='#CFCFCF', c=colors, cmap='cool', lw = 0) divider = make_axes_locatable(ax) # plt.title('Jaccard similarity with New York') cax = divider.append_axes("right", size="5%", pad=0.05) plt.colorbar(sc, cax=cax) plt.show() # plt.savefig('../images/coverage_examples/%s_%s.png'%(hashtag, type)) plt.clf() break
def significant_nei_utm_ids(): output_folder = fld_google_drive_data_analysis%GeneralMethods.get_method_id()+'/%s.png' for i, data in enumerate(FileIO.iterateJsonFromFile(f_significant_nei_utm_ids, remove_params_dict=True)): utm_lat_long = UTMConverter.getLatLongUTMIdInLatLongForm(data['utm_id']) nei_utm_lat_longs = map( lambda nei_utm_id: UTMConverter.getLatLongUTMIdInLatLongForm(nei_utm_id), data['nei_utm_ids'] ) if nei_utm_lat_longs: output_file = output_folder%('%s_%s'%(utm_lat_long)) plotPointsOnWorldMap(nei_utm_lat_longs, blueMarble=False, bkcolor='#CFCFCF', lw = 0, color = '#EA00FF', alpha=1.) _, m = plotPointsOnWorldMap([utm_lat_long], blueMarble=False, bkcolor='#CFCFCF', lw = 0, color = '#2BFF00', s = 40, returnBaseMapObject=True, alpha=1.) for nei_utm_lat_long in nei_utm_lat_longs: m.drawgreatcircle(utm_lat_long[1], utm_lat_long[0], nei_utm_lat_long[1], nei_utm_lat_long[0], color='#FFA600', lw=1.5, alpha=1.0) print 'Saving %s'%(i+1) savefig(output_file)
def plotDimensionsUpdateFrequencyEstimation(self, returnAxisValuesOnly=True): ''' numberOfTimeUnits=10*24*12 Experts stream 12 Houston stream 2 ''' dataDistribution = defaultdict(list) for line in FileIO.iterateJsonFromFile(self.dimensionsUpdateFrequencyFile): for k, v in line[ParameterEstimation.dimensionsUpdateFrequencyId].iteritems(): k = int(k) / self.timeUnitInSeconds.seconds if k not in dataDistribution: dataDistribution[k] = [0., 0.] dataDistribution[k][0] += v; dataDistribution[k][1] += 1 x, y = [], []; [(x.append(k), y.append((dataDistribution[k][0] / dataDistribution[k][1]))) for k in sorted(dataDistribution)] x1, y1 = [], []; [(x1.append(k), y1.append((dataDistribution[k][0] / dataDistribution[k][1]) / k)) for k in sorted(dataDistribution)] x = x[:numberOfTimeUnits]; y = y[:numberOfTimeUnits]; x1 = x1[:numberOfTimeUnits]; y1 = y1[:numberOfTimeUnits] def subPlot(id): plt.subplot(id) inactivityCorordinates = max(zip(x1, y1), key=itemgetter(1)) plt.semilogx(x1, y1, '-', color=self.stream_settings['plot_color'], label=getLatexForString(self.stream_settings['plot_label'] + ' (Update frequency=%d TU)' % inactivityCorordinates[0]), lw=2) plt.subplot(id).yaxis.set_major_formatter(FuncFormatter(lambda x, i: '%0.1f' % (x / 10. ** 3))) plt.semilogx([inactivityCorordinates[0]], [inactivityCorordinates[1]], 'o', alpha=0.7, color='r') plt.subplot(id).yaxis.set_major_formatter(FuncFormatter(lambda x, i: '%0.1f' % (x / 10. ** 3))) plt.yticks((min(y1), max(y1))) print self.stream_settings['plot_label'], inactivityCorordinates[0] plt.subplot(311) plt.title(getLatexForString('Dimensions update frequency estimation')) plt.semilogx(x, y, '-', color=self.stream_settings['plot_color'], label=getLatexForString(self.stream_settings['plot_label']), lw=2) plt.subplot(311).yaxis.set_major_formatter(FuncFormatter(lambda x, i: '%0.1f' % (x / 10. ** 5))) plt.text(0.0, 1.01, getLatexForString('10^5'), transform=plt.gca().transAxes) plt.ylabel(getLatexForString('\# of decayed dimensions')) if self.stream_settings['stream_id'] == 'experts_twitter_stream': subPlot(312) else: subPlot(313); plt.xlabel(getLatexForString(xlabelTimeUnits)) plt.ylabel(getLatexForString('Rate of DD (10^3)')) plt.legend(loc=3) if returnAxisValuesOnly: plt.show()
def plotGrowthOfPhrasesInTime(self, returnAxisValuesOnly=True): ''' This plot tells us the time when the number of phrases in the stream stablizes. Consider the time after we have seen maximum phrases to determine dimensions. But, if these phrases increase linearly with time, it shows that we have infinte dimensions and hence this motivates us to have a way to determine number of dimensions. numberOfTimeUnits=10*24*12 ''' x, y = [], [] [(x.append(getDateTimeObjectFromTweetTimestamp(line['time_stamp'])), y.append(line['total_number_of_phrases'])) for line in FileIO.iterateJsonFromFile(self.dimensionsEstimationFile)] x = x[:numberOfTimeUnits] y = y[:numberOfTimeUnits] plt.subplot(111).yaxis.set_major_formatter( FuncFormatter(lambda x, i: '%0.1f' % (x / 10.**6))) plt.text(0.0, 1.01, getLatexForString('10^6'), transform=plt.gca().transAxes) plt.ylabel(getLatexForString('\# of dimensions')), plt.xlabel( getLatexForString(xlabelTimeUnits)), plt.title( getLatexForString( 'Growth in dimensions with increasing time.')) plt.plot(y, color=self.stream_settings['plot_color'], label=getLatexForString(self.stream_settings['plot_label']), lw=2) plt.legend(loc=4) if returnAxisValuesOnly: plt.show()
def trendCurves(iterationData=None, experimentFileName=None): if iterationData: currentTimeStep, _, currentTopics, _, finalCall, conf = iterationData experimentFileName = conf['experimentFileName'] if not finalCall: topicDistribution = dict((str(topic.id), {'total': topic.totalCount, 'timeStep': topic.countDistribution[currentTimeStep]}) for topic in currentTopics) # print currentTimeStep FileIO.writeToFileAsJson({'t':currentTimeStep, 'topics':topicDistribution}, experimentFileName) else: iterationInfo = {'trending_topics': [topic.id for topic in currentTopics if topic.stickiness>=stickinessLowerThreshold], 'topic_colors': dict((str(topic.id), topic.color) for topic in currentTopics), 'conf': conf} del iterationInfo['conf']['spamDectectionMethod'] FileIO.writeToFileAsJson(iterationInfo, experimentFileName) else: topicsDataX = defaultdict(list) topicsDataY = defaultdict(list) for data in FileIO.iterateJsonFromFile(experimentFileName): if 'conf' not in data: for topic in data['topics']: topicsDataX[topic].append(data['t']), topicsDataY[topic].append(data['topics'][topic]['timeStep']) else: topicColorMap=data['topic_colors']; trendingTopics=data['trending_topics'] for topic in topicsDataX: plt.fill_between(topicsDataX[topic], topicsDataY[topic], color=topicColorMap[str(topic)], alpha=1.0) plt.figure() for topic in trendingTopics: plt.fill_between(topicsDataX[str(topic)], topicsDataY[str(topic)], color=topicColorMap[str(topic)], alpha=1.0) plt.ylabel('Number of Contents', fontsize=16, fontweight='bold') plt.show()
def plotClusteringSpeed(saveFig=True): dataToPlot = { 'k_means': { 'x': [], 'y': [] }, 'mr_k_means': { 'x': [], 'y': [] }, 'streaming_lsh': { 'x': [], 'y': [] } } for data in FileIO.iterateJsonFromFile(TweetsFile.combined_stats_file): for k in plotSettings: dataToPlot[k]['x'].append(data[k]['no_of_documents']) dataToPlot[k]['y'].append(data[k]['iteration_time']) for k in plotSettings: plt.loglog(dataToPlot[k]['x'], dataToPlot[k]['y'], label=plotSettings[k]['label'], color=plotSettings[k]['color'], lw=2) plt.legend(loc=4) plt.xlabel(getLatexForString('\# of documents')) plt.ylabel(getLatexForString('Running time (s)')) plt.title( getLatexForString( 'Running time comparsion for Streaing LSH with k-Means')) plt.xlim(xmax=500000) # plt.show() if saveFig: plt.savefig('speedComparisonWithKMeans.pdf')
def ef_plot(): output_file = fld_data_analysis_results%GeneralMethods.get_method_id()+'.png' data = [d for d in FileIO.iterateJsonFromFile(f_hashtag_spatial_metrics, remove_params_dict=True)] ltuo_hashtag_and_entropy_and_focus = map(itemgetter('hashtag', 'entropy', 'focus'), data) mf_norm_focus_to_entropies = defaultdict(list) for _, entropy, (_, focus) in ltuo_hashtag_and_entropy_and_focus: mf_norm_focus_to_entropies[round(focus, 2)].append(entropy) plt.figure(num=None, figsize=(6,3)) x_focus, y_entropy = zip(*[(norm_focus, np.mean(entropies)) for norm_focus, entropies in mf_norm_focus_to_entropies.iteritems() if len(entropies)>0]) plt.subplots_adjust(bottom=0.2, top=0.9, wspace=0, hspace=0) plt.scatter(x_focus, y_entropy, s=50, lw=0, c='k') plt.xlim(xmin=-0.1, xmax=1.1) plt.ylim(ymin=-1, ymax=9) plt.xlabel('Mean hashtag focus') plt.ylabel('Mean hashtag entropy') plt.grid(True) savefig(output_file) ltuo_hashtag_and_r_entropy_and_focus =\ sorted(ltuo_hashtag_and_entropy_and_focus, key=itemgetter(1), reverse=True) ltuo_hashtag_and_r_entropy_and_s_focus = sorted(ltuo_hashtag_and_r_entropy_and_focus, key=itemgetter(2)) hashtags = zip(*ltuo_hashtag_and_r_entropy_and_s_focus)[0] print list(hashtags[:20]) print list(reversed(hashtags))[:20]
def getStats(spotsFile, userToLocationVector): lidToSpotIdMap, userToSpotIdMap, spotMap, spotsWithUsersFile = {}, {}, defaultdict(dict), spotsFile + "_users" for spot in FileIO.iterateJsonFromFile(spotsWithUsersFile): for location, _ in spot["lids"]: lidToSpotIdMap[getLidFromLocation(location)] = spot["id"] for user in spot["users"]: userToSpotIdMap[user] = spot["id"] observedUsers = set() for userVector in userToLocationVector: user = userVector["user"] assert user not in observedUsers if user in userToSpotIdMap: assignment = [ [lidToSpotIdMap[lid]] * userVector["locations"][lid] for lid in userVector["locations"] if lid in lidToSpotIdMap ] spotMap[userToSpotIdMap[user]][user] = [item for t in assignment for item in t] observedUsers.add(user) accuracyList = [] for spotId, userMap in spotMap.iteritems(): totalAssignments, wrongAssignments = 0.0, 0.0 for user in userMap: for a in userMap[user]: if a != spotId: wrongAssignments += 1 totalAssignments += 1 accuracyList.append(wrongAssignments / totalAssignments) return { "accuracy": np.mean(accuracyList), "total_locations": len(lidToSpotIdMap), "total_users": len(userToSpotIdMap), }
def plotICDFDimensionsInactivityThreshold(self, returnAxisValuesOnly=True): ''' Plot P(in_actiivty > threshold timeunit) Find time unit at which probability is low. Experts stream 0.25 129 Houston stream 0.25 144 ''' dataX, dataY, total = set(), defaultdict(list), [] for line in list( FileIO.iterateJsonFromFile(self.dimensionInActivityTimeFile)): data = dict((int(k), v) for k, v in line[ ParameterEstimation.dimensionInActivityTimeId].iteritems()) total.append(sum(data.values())) for i in data: dataY[i].append(data[i]) dataX.add(i) totalInstancesObserved = float(sum(total)) x = sorted(dataX) y = getInverseCumulativeDistribution( [sum(dataY[k]) / totalInstancesObserved for k in x]) plt.plot(x, y, label=getLatexForString(self.stream_settings['plot_label']), color=self.stream_settings['plot_color'], lw=2) plt.ylabel( r'$P\ (\ inactivity\ duration\ \geq\ \ inactivity\ duration\ threshold )$' ), plt.xlabel( getLatexForString('Inactivity duration threshold')), plt.title( getLatexForString('Inactivity analysis for dimensions.')) plt.legend() if returnAxisValuesOnly: plt.show()
def iid_vs_cumulative_distribution_and_peak_distribution(): TIME_UNIT_IN_SECONDS = 10.*60. output_file_format = fld_data_analysis_results%GeneralMethods.get_method_id()+'/%s.png' ltuo_iid_and_interval_stats = [data for data in FileIO.iterateJsonFromFile(f_iid_spatial_metrics, remove_params_dict=True)] ltuo_s_iid_and_interval_stats = sorted(ltuo_iid_and_interval_stats, key=itemgetter(0)) ltuo_s_iid_and_tuo_is_peak_and_cumulative_percentage_of_occurrences = [(data[0], (data[1][0], data[1][2])) for data in ltuo_s_iid_and_interval_stats] total_peaks = sum([data[1][0] for data in ltuo_s_iid_and_tuo_is_peak_and_cumulative_percentage_of_occurrences])+0.0 x_iids = [] y_is_peaks = [] z_cumulative_percentage_of_occurrencess = [] for (iid, (is_peak, cumulative_percentage_of_occurrences)) in ltuo_s_iid_and_tuo_is_peak_and_cumulative_percentage_of_occurrences[:100]: print (iid, (is_peak, cumulative_percentage_of_occurrences)) x_iids.append((iid+1)*TIME_UNIT_IN_SECONDS/60) y_is_peaks.append(is_peak/total_peaks) z_cumulative_percentage_of_occurrencess.append(cumulative_percentage_of_occurrences) plt.figure(num=None, figsize=(4.3,3)) plt.subplots_adjust(bottom=0.2, top=0.9, wspace=0, hspace=0) plt.plot(x_iids, y_is_peaks, marker='o', c='k') plt.ylabel('Distribution of hashtags') plt.xlabel('Hashtag peak (minutes)') plt.grid(True) plt.xlim(xmax=600) savefig(output_file_format%'peaks') plt.clf() plt.figure(num=None, figsize=(6,3)) plt.subplots_adjust(bottom=0.2, top=0.9, wspace=0, hspace=0) plt.plot(x_iids, z_cumulative_percentage_of_occurrencess, lw=0, marker='o', c='k') # plt.xlabel('Minutes') plt.ylabel('CDF of occurrences') plt.xlabel('Time (Minutes)') plt.grid(True) plt.xlim(xmax=600) savefig(output_file_format%'cdf_occurrences_peak')
def drawKMLsForUserCooccurenceGraph(minEdgeWeight=30): kml = SpotsKML() i=1 for edge in FileIO.iterateJsonFromFile(locationGraph): if edge['w']>=minEdgeWeight: kml.addLine(getLocationPairs(edge['e']), description=str(edge['w'])); i+=1 if i==10000: break kml.write(userBasedSpotsKmlsFolder+'%s.kml'%minEdgeWeight)
def iterateExpertClusters(startingDay=datetime(2011,3,19), endingDay=datetime(2011,3, 30)): # def iterateExpertClusters(startingDay=datetime(2011,3,19), endingDay=datetime(2011,4,7)): while startingDay<=endingDay: for line in FileIO.iterateJsonFromFile(experts_twitter_stream_settings.lsh_clusters_folder+FileIO.getFileByDay(startingDay)): currentTime = getDateTimeObjectFromTweetTimestamp(line['time_stamp']) for clusterMap in line['clusters']: yield (currentTime, TwitterCrowdsSpecificMethods.getClusterFromMapFormat(clusterMap)) startingDay+=timedelta(days=1)
def getClusteringQuality(): ''' no_of_documents: 300000 k_means f1, p, r ['(0.95 0.04)', '(0.95 0.04)', '(0.95 0.04)'] purity (0.95 0.04) nmi (0.94 0.04) streaming_lsh f1, p, r ['(0.67 0.01)', '(0.71 0.01)', '(0.64 0.02)'] purity (0.96 0.00) nmi (0.87 0.00) ''' del plotSettings['mr_k_means']; del plotSettings['default_streaming_lsh'] speedStats = dict([(k, {'f1': [], 'nmi': [], 'purity': []}) for k in plotSettings]) for data in FileIO.iterateJsonFromFile(TweetsFile.combined_stats_file): for k in speedStats: for metric in speedStats['k_means']: speedStats[k][metric].append(data[k][metric]) # Adding this because final value of f1 is 0 instead of tuple at 300K documents. speedStats['k_means']['f1'][-1]=[0.,0.,0.] dataForPlot = dict([(k, []) for k in plotSettings]) for k, v in speedStats.iteritems(): print k for k1,v1 in v.iteritems(): if type(v1[0])!=type([]): print k1, '(%0.2f %0.2f)'%(np.mean(v1), np.var(v1)); dataForPlot[k]+=[np.mean(v1)] else: print k1, ['(%0.2f %0.2f)'%(np.mean(z),np.var(z)) for z in zip(*v1)]; dataForPlot[k]+=[np.mean(z) for z in zip(*v1)]
def sortFile(fileName): dataToWrite = sorted(FileIO.iterateJsonFromFile(fileName), key=lambda l: dateutil.parser.parse(l['CreationDate'])) print 'Sorting file', fileName f = open('%s_sorted'%fileName, 'w') for line in dataToWrite: f.write(cjson.encode(line)+'\n') os.system('mv %s_sorted %s'%(fileName, fileName)) f.close()
def print_dense_utm_ids(): ''' Prints list of dense utm_ids. ''' print [utm_object['utm_id'] for utm_object in FileIO.iterateJsonFromFile( f_hashtags_by_utm_id, remove_params_dict=True)]
def analyzeJustifyExponentialDecay(self): global evaluation experimentsData = {JustifyExponentialDecay.with_decay: {}, JustifyExponentialDecay.without_decay: {}} for data in FileIO.iterateJsonFromFile(JustifyExponentialDecay.stats_file): experimentsData[data['iteration_parameters']['type']][getDateTimeObjectFromTweetTimestamp(data['iteration_parameters']['current_time'])]=data['clusters'] qualityData = [] for k1, k2 in zip(sorted(experimentsData[JustifyExponentialDecay.with_decay]), sorted(experimentsData[JustifyExponentialDecay.without_decay])): qualityData.append((k1, evaluation.getEvaluationMetrics(experimentsData[JustifyExponentialDecay.with_decay][k1], None, None)['purity']-evaluation.getEvaluationMetrics(experimentsData[JustifyExponentialDecay.without_decay][k1], None, None)['purity'])) keyTime = sorted(qualityData, key=itemgetter(1))[-1][0] clusterWithDecay = [i for i in experimentsData[JustifyExponentialDecay.with_decay][keyTime] if len(i)>=3] clusterWithOutDecay = [i for i in experimentsData[JustifyExponentialDecay.without_decay][keyTime] if len(i)>=3] # for c in clusterWithDecay: # print c, [evaluation.expertsToClassMap[i.lower()] for i in c] interestedCluster = set(['Zap2it', 'ESPNAndyKatz', 'comingsoonnet', '950KJR', 'ginasmith888', 'UKCoachCalipari', 'SportsFanz', 'David_Henrie']) for c in clusterWithOutDecay: if len(set(c).intersection(interestedCluster))>0: # print c, [evaluation.expertsToClassMap[i.lower()] for i in c] setString = ', '.join(['%s (%s)'%(i, evaluation.expertsToClassMap[i.lower()]) for i in sorted(c)]).replace(' ', '\\ ').replace('_', '\\_') print keyTime, '&', setString, '\\\\' clustersDiscoveredEarlierByDecay = {} for kt in sorted(experimentsData[JustifyExponentialDecay.with_decay]): for c in experimentsData[JustifyExponentialDecay.with_decay][kt]: c=sorted(c) if len(set(c).intersection(interestedCluster))>0: classes = [evaluation.expertsToClassMap[i.lower()] for i in c if i.lower() in evaluation.expertsToClassMap] if sorted([(k, len(list(g))/float(len(classes))) for k,g in groupby(sorted(classes))], key=itemgetter(1))[-1][1]>0.7: if kt>datetime(2011,3,19) and kt<=keyTime: clustersDiscoveredEarlierByDecay[kt]=c observedStrings = set() for k in sorted(clustersDiscoveredEarlierByDecay): setString = ', '.join(['%s (%s)'%(i, evaluation.expertsToClassMap[i.lower()]) for i in sorted(clustersDiscoveredEarlierByDecay[k])]).replace(' ', '\\ ').replace('_', '\\_') if setString not in observedStrings: print k, '&', setString, '\\\\'; observedStrings.add(setString)
def load_mf_hashtag_to_ltuo_point_and_occurrence_time(WINDOW_IN_MINUTES): def is_unicode(hashtag): try: hashtag.decode('ascii') except Exception: return False return True mf_hashtag_to_ltuo_point_and_occurrence_time = defaultdict(list) # Subtracting because stream appears to be delayed by an hour # dt_current_time = datetime.fromtimestamp(time.mktime(time.gmtime(time.time()))) - timedelta(hours=1) dt_current_time = datetime.fromtimestamp(time.mktime(time.gmtime(time.time()))) td_interval = timedelta(seconds=INTERVAL_IN_MINUTES * 60) td_window = timedelta(seconds= WINDOW_IN_MINUTES * 60) dt_next_time = dt_current_time - td_window while dt_next_time < dt_current_time: f_input = GetOutputFile(dt_next_time) if os.path.exists(f_input): print 'Processing:', f_input for checkin in FileIO.iterateJsonFromFile(f_input): for hashtag, point_and_occurrence_time in \ TweetStreamDataProcessing._ParseHashtagObjects(checkin): if hashtag not in BLOCKED_HASHTAGS and is_unicode(hashtag): mf_hashtag_to_ltuo_point_and_occurrence_time[hashtag].append(point_and_occurrence_time) dt_next_time += td_interval # return mf_hashtag_to_ltuo_point_and_occurrence_time # Modifying this code to normalize lattice and time for all analysis mf_hashtag_to_normalized_ltuo_point_and_occurrence_time = {} for hashtag, ltuo_point_and_occurrence_time in \ mf_hashtag_to_ltuo_point_and_occurrence_time.iteritems(): ltuo_point__lattice__normalized_occurrence_time = \ SpatialAnalysisAlgorithms._get_ltuo_point_and_lattice_and_normalized_occurrence_time(ltuo_point_and_occurrence_time) ltuo_point__lattice__normalized_occurrence_time = SpatialAnalysisAlgorithms._get_valid_occurrences(ltuo_point__lattice__normalized_occurrence_time) mf_hashtag_to_normalized_ltuo_point_and_occurrence_time[hashtag] \ = [(lattice, normalized_occurrence_time) for _, lattice, normalized_occurrence_time in ltuo_point__lattice__normalized_occurrence_time] return mf_hashtag_to_normalized_ltuo_point_and_occurrence_time
def writeClusterKML(): kml = SpotsKML() outputKMLFile='%s/clusters.kml'%placesAnalysisFolder%place['name'] for data in FileIO.iterateJsonFromFile(placesUserClusterFeaturesFile%place['name']): clusterId, color, features = data kml.addLocationPointsWithTitles([(getLocationFromLid(f[0].replace('_', ' ')), f[2]) for f in features[:noOfFeatures]], color=color) FileIO.createDirectoryForFile(outputKMLFile) kml.write(outputKMLFile)
def loadExperimentsData(experimentsData, file): for data in FileIO.iterateJsonFromFile(file): if data["purity"] > 0 and data["purity"] < 1: experimentsData[data["iteration_parameters"]["type"]]["iteration_time"].append(data["iteration_time"]) experimentsData[data["iteration_parameters"]["type"]]["quality"].append(data["purity"]) experimentsData[data["iteration_parameters"]["type"]]["total_clusters"].append( data["iteration_parameters"]["total_clusters"] )
def temp(hashtag='blackparentsquotes'): for hashtagObject in FileIO.iterateJsonFromFile('/mnt/chevron/kykamath/data/geo/hashtags/analysis/all_world/2_11/hashtagsWithoutEndingWindow'): # print hashtagObject['h'] if hashtagObject['h']==hashtag: print hashtagObject['h'] occsDistributionInTimeUnits = getOccurranceDistributionInEpochs(hashtagObject['oc'], timeUnit=5, fillInGaps=True, occurancesCount=False) # plt.plot_date() exit()
def writeHashtagsFile(): hashtags = [] for hashtagObject in FileIO.iterateJsonFromFile('/mnt/chevron/kykamath/data/geo/hashtags/analysis/all_world/2_11/hashtagsWithoutEndingWindow'): print hashtagObject.keys() exit() hashtags.append(hashtagObject['h']) hashtags=sorted(hashtags) for h in hashtags: FileIO.writeToFile(unicode(h).encode('utf-8'), 'hashtags')
def getClusteringQuality(): ''' no_of_documents: 300000 k_means f1, p, r ['(0.95 0.04)', '(0.95 0.04)', '(0.95 0.04)'] purity (0.95 0.04) nmi (0.94 0.04) streaming_lsh f1, p, r ['(0.67 0.01)', '(0.71 0.01)', '(0.64 0.02)'] purity (0.96 0.00) nmi (0.87 0.00) ''' del plotSettings['mr_k_means'] del plotSettings['default_streaming_lsh'] speedStats = dict([(k, { 'f1': [], 'nmi': [], 'purity': [] }) for k in plotSettings]) for data in FileIO.iterateJsonFromFile(TweetsFile.combined_stats_file): for k in speedStats: for metric in speedStats['k_means']: speedStats[k][metric].append(data[k][metric]) # Adding this because final value of f1 is 0 instead of tuple at 300K documents. speedStats['k_means']['f1'][-1] = [0., 0., 0.] dataForPlot = dict([(k, []) for k in plotSettings]) for k, v in speedStats.iteritems(): print k for k1, v1 in v.iteritems(): if type(v1[0]) != type([]): print k1, '(%0.2f %0.2f)' % (np.mean(v1), np.var(v1)) dataForPlot[k] += [np.mean(v1)] else: print k1, [ '(%0.2f %0.2f)' % (np.mean(z), np.var(z)) for z in zip(*v1) ] dataForPlot[k] += [np.mean(z) for z in zip(*v1)] ind, width = np.arange(5), 0.1 rects, i = [], 0 for k in dataForPlot: rects.append( plt.bar(ind + i * width, dataForPlot[k], width, color=plotSettings[k]['color'])) i += 1 plt.ylabel(getLatexForString('Score')) plt.title( getLatexForString( 'Clustering quality comparison for Streaming LSH with k-Means') ) plt.xticks(ind + width, ('$F$', '$Precision$', '$Recall$', '$Purity$', '$NMI$')) plt.legend([r[0] for r in rects], [plotSettings[k]['label'] for k in plotSettings], loc=4) plt.show()
def getRandomLocationNames(file, **conf): data = defaultdict(list) for d in FileIO.iterateJsonFromFile(file): data[d['location_db_mad']].append(d['location']) for k in sorted(data): print k, for i in range(5): venue = venuesCollection.find_one({'lid':random.choice(data[k])}) if venue: print unicode(venue['n']).encode("utf-8")+', ', print
def performanceWithSpamFilteringForLatestMessages(generateData): experimentData = defaultdict(dict) for iteration in range(10): # for spammerPercentage in range(1,21): ## spammerPercentage = 20 # spammerPercentage = spammerPercentage*0.05 # for spammerPercentage in range(1,11): # spammerPercentage = spammerPercentage*0.02 # for spammerPercentage in range(1,201): # spammerPercentage = spammerPercentage* 0.005 l1 = [spammerPercentage* 0.001 for spammerPercentage in range(1,51)] l2 = [spammerPercentage* 0.05 for spammerPercentage in range(1,21)] l3 = [0.01]+l2 for spammerPercentage in l1: experimentFileName = spamModelFolder+'performanceWithSpamFilteringForLatestMessages/%s/%0.3f'%(iteration,spammerPercentage) print experimentFileName if generateData: model = MixedUsersModel() conf = {'model': model, 'numberOfTimeSteps': 10, 'addUsersMethod': User.addUsersUsingRatio, 'analysisMethods': [(Analysis.measureRankingQuality, 1)], 'ratio': {'normal': 1-spammerPercentage, 'spammer': spammerPercentage}, 'rankingMethods':[RankingModel.latestMessages, RankingModel.latestMessagesSpamFiltered], 'experimentFileName': experimentFileName, # 'noOfPayloadsPerSpammer': 1, 'noOfTopics': 10 } # conf = {'model': model, 'numberOfTimeSteps': 10, 'addUsersMethod': User.addUsersUsingRatio, 'analysisMethods': [(Analysis.measureRankingQuality, 1)], 'ratio': {'normal': 1-spammerPercentage, 'spammer': spammerPercentage}, # 'rankingMethods':[RankingModel.latestMessages, RankingModel.latestMessagesDuplicatesRemoved, RankingModel.popularMessages], # 'experimentFileName': experimentFileName} GeneralMethods.runCommand('rm -rf %s'%experimentFileName);run(**conf) else: tempData = defaultdict(list) for data in FileIO.iterateJsonFromFile(experimentFileName): for ranking_id in data['spammmess']: tempData[ranking_id]+=data['spammmess'][ranking_id] experimentData[iteration][spammerPercentage]=tempData if not generateData: realDataY = defaultdict(dict) for iteration in experimentData: dataY = defaultdict(list) dataX = [] for perct in sorted(experimentData[iteration]): dataX.append(perct) for ranking_id, values in experimentData[iteration][perct].iteritems(): dataY[ranking_id].append(np.mean(values)) dataX=sorted(dataX) for ranking_id in dataY: for x, y in zip(dataX, dataY[ranking_id]): if x not in realDataY[ranking_id]: realDataY[ranking_id][x]=[] realDataY[ranking_id][x].append(y) for ranking_id in dataY: plt.plot(dataX, [np.mean(realDataY[ranking_id][x]) for x in dataX], label=labels[ranking_id], lw=1, marker=RankingModel.marker[ranking_id]) plt.xlabel('Percentage of Spammers', fontsize=16, fontweight='bold') plt.ylabel('Spamness', fontsize=16, fontweight='bold') # plt.title('Performance with spam filtering') plt.legend(loc=2) # plt.show() plt.xlim(xmax=0.05) plt.savefig('performanceWithSpamFilteringForLatestMessages.png') plt.clf()
def loadExperimentsData(experimentsData, file): for data in FileIO.iterateJsonFromFile(file): if data['purity'] > 0 and data['purity'] < 1: experimentsData[data['iteration_parameters'] ['type']]['iteration_time'].append( data['iteration_time']) experimentsData[data['iteration_parameters'] ['type']]['quality'].append(data['purity']) experimentsData[data['iteration_parameters'] ['type']]['total_clusters'].append( data['iteration_parameters']['total_clusters'])
def plotDimensionsEstimation(self, returnAxisValuesOnly=True): def calculateDimensionsFor(params, percentageOfNewDimensions): ''' numberOfTimeUnits=10*24*12 Experts stream [ 1.17707899e+03 1.03794580e+00] 76819 Houston stream [ 2.73913900e+03 1.02758516e+00] 195731 ''' print getSmallestPrimeNumberGreaterThan( int( CurveFit.inverseOfDecreasingExponentialFunction( params, percentageOfNewDimensions))) dataDistribution = defaultdict(list) for line in FileIO.iterateJsonFromFile(self.dimensionsEstimationFile): for k, v in line[ ParameterEstimation.dimensionsEstimationId].iteritems(): k = int(k) if k not in dataDistribution: dataDistribution[k] = [0., 0.] dataDistribution[k][0] += v dataDistribution[k][1] += 1 x, y = [], [] [(x.append(k), y.append((dataDistribution[k][0] / dataDistribution[k][1]) / k)) for k in sorted(dataDistribution) if k > 1000] x, y = x[:numberOfTimeUnits], y[:numberOfTimeUnits] exponentialCurveParams = CurveFit.getParamsAfterFittingData( x, y, CurveFit.decreasingExponentialFunction, [1., 1.]) print self.stream_settings[ 'plot_label'], exponentialCurveParams, calculateDimensionsFor( exponentialCurveParams, 0.01) plt.ylabel(getLatexForString('\% of decaying dimensions')), plt.xlabel( getLatexForString('\# of dimensions') ), plt.title( getLatexForString( 'Dimension stability with increasing number of dimensions.')) plt.semilogy( x, y, 'o', color=self.stream_settings['plot_color'], label=getLatexForString(self.stream_settings['plot_label']) + getLatexForString(' (%0.2fx^{-%0.2f})') % (exponentialCurveParams[0], exponentialCurveParams[1]), lw=2) plt.semilogy(x, CurveFit.getYValues( CurveFit.decreasingExponentialFunction, exponentialCurveParams, x), color=self.stream_settings['plot_color'], lw=2) plt.legend() if returnAxisValuesOnly: plt.show()
def iterateUserDocuments(fileName): dataForAggregation = defaultdict(Vector) textToIdMap = defaultdict(int) for tweet in FileIO.iterateJsonFromFile(fileName): textVector = TwitterCrowdsSpecificMethods.convertTweetJSONToMessage( tweet, **default_experts_twitter_stream_settings).vector textIdVector = Vector() for phrase in textVector: if phrase not in textToIdMap: textToIdMap[phrase] = str(len(textToIdMap)) textIdVector[textToIdMap[phrase]] = textVector[phrase] dataForAggregation[tweet['user'] ['screen_name'].lower()] += textIdVector for k, v in dataForAggregation.iteritems(): yield k, v
def plotClusteringQuality(): del plotSettings['ssa_mr'] speedStats = dict([(k, { 'f1': [], 'nmi': [], 'purity': [] }) for k in plotSettings]) for data in FileIO.iterateJsonFromFile(TweetsFile.stats_file): for k in speedStats: for metric in speedStats['ssa']: speedStats[k][metric].append(data[k][metric]) dataForPlot = dict([(k, []) for k in plotSettings]) for k, v in speedStats.iteritems(): print k for k1, v1 in v.iteritems(): if type(v1[0]) != type([]): print k1, '(%0.2f %0.2f)' % (np.mean(v1), np.var(v1)) dataForPlot[k] += [np.mean(v1)] else: print k1, [ '(%0.2f %0.2f)' % (np.mean(z), np.var(z)) for z in zip(*v1) ] dataForPlot[k] += [np.mean(z) for z in zip(*v1)] ind, width = np.arange(5), 0.1 rects, i = [], 0 for k in dataForPlot: rects.append( plt.bar(ind + i * width, dataForPlot[k], width, color=plotSettings[k]['color'])) i += 1 plt.ylabel(getLatexForString('Score')) plt.title( getLatexForString( 'Clustering quality comparison for Streaming LSH with SSA')) plt.xticks(ind + width, ('$F$', '$Precision$', '$Recall$', '$Purity$', '$NMI$')) plt.legend([r[0] for r in rects], [plotSettings[k]['label'] for k in plotSettings], loc=4) # plt.show() plt.savefig('qualityComparisonWithSSA.pdf')
def plotPercentageOfDimensionsWithinALag(self, returnAxisValuesOnly=True): ''' This gives us the percentage of phrases we can loose everytime we prune phrases. Measures the percentage of dimensions having lag less than TU. So at the end of 10th day, almost y% of phrases can be removed. With some probabiity that it will not occure again. numberOfTimeUnits=10*24*12 With 75% probability. Experts stream [ 0.0097055 0.81888514] 107 0.554497397565 Houston stream [ 0.00943499 0.825918 ] 126 0.487757815615 With 90% probability. Experts stream [ 0.0097055 0.81888514] 223 0.187150798756 Houston stream [ 0.00943499 0.825918 ] 228 0.164007589276 ''' def calculatePercentageOfDecayedPhrasesFor(params, timeUnit): return 1 - CurveFit.increasingExponentialFunction(params, timeUnit) dataDistribution = {} currentTimeUnit = 0 for data in list(FileIO.iterateJsonFromFile(self.dimensionInActivityTimeFile))[:numberOfTimeUnits]: totalDimensions = float(sum(data['phrases_lag_distribution'].values())) tempArray = [] for k, v in data['phrases_lag_distribution'].iteritems(): k = int(k) if k not in dataDistribution: dataDistribution[k] = [0] * numberOfTimeUnits dataDistribution[k][currentTimeUnit] = v / totalDimensions tempArray.append(v / totalDimensions) currentTimeUnit += 1 x = sorted(dataDistribution) y = getCumulativeDistribution([np.mean(dataDistribution[k]) for k in x]) params = CurveFit.getParamsAfterFittingData(x, y, CurveFit.increasingExponentialFunction, [1., 1.]) print self.stream_settings['plot_label'], params, def subPlot(id, timeUnit): plt.subplot(id) print timeUnit, calculatePercentageOfDecayedPhrasesFor(params, timeUnit) plt.plot(x, y, 'o', label=getLatexForString(self.stream_settings['plot_label']) + getLatexForString(' (%0.2fx^{%0.2f})') % (params[0], params[1]), color=self.stream_settings['plot_color']) plt.plot(x, CurveFit.getYValues(CurveFit.increasingExponentialFunction, params, x), color=self.stream_settings['plot_color'], lw=2) if self.stream_settings['stream_id'] == 'experts_twitter_stream': subPlot(111, 107); plt.title(getLatexForString('Percentage of phrases within a lag')) else: subPlot(111, 126); plt.xlabel(getLatexForString(xlabelTimeUnits)) plt.ylabel(r'$\%\ of\ phrases\ with\ lag\ \leq\ TU$') plt.legend(loc=4) if returnAxisValuesOnly: plt.show()
def plotICDFClustersLagDistribution(self, returnAxisValuesOnly=True): ''' Experts stream 0.25 199 Houston stream 0.25 152 ''' self.stream_settings['%s_file' % ClusteringParametersEstimation.clusterLagDistributionId] = self.stream_settings['parameter_estimation_folder'] + ClusteringParametersEstimation.clusterLagDistributionId dataX, dataY, total = set(), defaultdict(list), [] for line in list(FileIO.iterateJsonFromFile(self.stream_settings['%s_file' % ClusteringParametersEstimation.clusterLagDistributionId])): print line.keys() data = dict((int(k), v) for k,v in line[ClusteringParametersEstimation.clusterLagDistributionId].iteritems()) total.append(sum(data.values())) for i in data: dataY[i].append(data[i]); dataX.add(i) totalInstancesObserved=float(sum(total)) x = sorted(dataX) y = getInverseCumulativeDistribution([sum(dataY[k])/totalInstancesObserved for k in x]) plt.plot(x, y, label=getLatexForString(self.stream_settings['plot_label']), color=self.stream_settings['plot_color'], lw=2) if self.stream_settings['plot_label']=='Houston stream': plt.plot([0,x[-1]], [1, 0], '--', color='#5AF522', lw=2) plt.ylabel(r'$P\ (\ inactivity\ duration\ \geq\ \ inactivity\ duration\ threshold )$'), plt.xlabel(getLatexForString('Inactivity duration threshold')), plt.title(getLatexForString('Inactivity analysis for crowds.')) plt.legend() if returnAxisValuesOnly: plt.show()
def plotJustifyDimensionsEstimation(self): runningTimeData, purityData = defaultdict(list), defaultdict(list) for data in FileIO.iterateJsonFromFile(JustifyDimensionsEstimation.stats_file): if data['iteration_parameters']['dimensions']<data['no_of_observed_dimensions']: no_of_dimensions = data['iteration_parameters']['dimensions'] runningTimeData[no_of_dimensions].append(data['iteration_time']), purityData[no_of_dimensions].append(data['purity']) plt.subplot(111) dataX, dataY = [], [] del purityData[169991]; del purityData[39989] plt.title('Impact of dimension estimation') for k in sorted(purityData): dataX.append(k), dataY.append(np.mean(purityData[k])) plt.semilogx(dataX, [0.96]*len(dataX), '--', label='Top n dimensions', color='#7109AA', lw=2) plt.semilogx(dataX, [np.mean(dataY)]*len(dataX), '--', color='#5AF522', lw=2) plt.semilogx(dataX, dataY, '-x', label='Fixed dimensions', color='#5AF522', lw=2) plt.ylim(0.8, 1.0) plt.xlim(7000, 203000) plt.xlabel('# of dimensions') plt.ylabel('Purity') plt.legend(loc=3) plt.savefig('justifyDimensionsEstimation.pdf')
def plotThresholdForDocumentToBeInCluster(self, statsFile): dataToPlot = dict(('%0.2f' % (t * 0.05), {'iteration_time':[], 'purity': [], 'nmi': []}) for t in range(1, 21)) for data in FileIO.iterateJsonFromFile(statsFile): threshold = '%0.2f' % data['settings']['threshold_for_document_to_be_in_cluster'] for k in dataToPlot[threshold]: dataToPlot[threshold][k] += [data['streaming_lsh'][k]] for t in dataToPlot: for k in dataToPlot[t]: dataToPlot[t][k] = np.mean(dataToPlot[t][k]) dataX = sorted([float(i) for i in dataToPlot])[:-1] print dataX # Plot iteration time. plt.subplot(211) plt.plot(dataX, [dataToPlot['%0.2f' % x]['iteration_time'] for x in dataX], lw=2, color='k') plt.ylabel(getLatexForString('Time (s)')) plt.title(getLatexForString('Estimation of \epsilon^\prime for Stream SSA')) plt.subplot(212) for metric, label, color in [('nmi', 'NMI', '#F60018'), ('purity', 'Purity', '#25D500')]: plt.plot(dataX, [dataToPlot['%0.2f' % x][metric] for x in dataX], label=label, color=color, lw=2) plt.ylabel(getLatexForString('Score')) plt.xlabel(getLatexForString('Similarity threshold (\epsilon^\prime)')) plt.legend(loc=4) plt.show()
def plotCDFDimensionsLagDistribution(self, returnAxisValuesOnly=True): ''' Inactivity time is the time after which there is a high probability that a dimension will not appear. Find time_unit that gives this probability. Cumulative distribution function (http://en.wikipedia.org/wiki/Cumulative_distribution_function) lag = time betweeen occurance of two dimensions (similar to inactivty_time) F(time_unit) = P(lag<=time_unit) time_unit = F_inv(P(lag<=time_unit)) Given P(inactivty_time>time_unit) determine time_unit as shown: P(inactivty_time<=time_unit) = 1 - P(inactivty_time>time_unit) inactivty_time = F_inv(P(inactivty_time<=time_unit)) numberOfTimeUnits=10*24*12 Experts stream [ 0.23250341 0.250209 ] 0.25 107 Houston stream [ 0.16948096 0.30751358] 0.25 126 Experts stream [ 0.23250341 0.250209 ] 0.1, 223 Houston stream [ 0.16948096 0.30751358] 0.1, 228 Compared to other vaues these values are pretty close to each other. This is expected. Irrespective of size of the streams, the phrases have the same lifetime and hence decay close to each other. ''' def calculateInActivityTimeFor(params, probabilityOfInactivity): return int(CurveFit.inverseOfIncreasingExponentialFunction(params, 1 - probabilityOfInactivity)) data = list(FileIO.iterateJsonFromFile(self.dimensionInActivityTimeFile))[numberOfTimeUnits] total = float(sum(data[ParameterEstimation.dimensionInActivityTimeId].values())) x = sorted(map(int, data[ParameterEstimation.dimensionInActivityTimeId].keys())) y = getCumulativeDistribution([data[ParameterEstimation.dimensionInActivityTimeId][str(i)] / total for i in x]) print len(x) exponentialCurveParams = CurveFit.getParamsAfterFittingData(x, y, CurveFit.increasingExponentialFunction, [1., 1.]) print self.stream_settings['plot_label'], exponentialCurveParams, calculateInActivityTimeFor(exponentialCurveParams, 0.1) plt.plot(x, y, 'o', label=getLatexForString(self.stream_settings['plot_label']) + getLatexForString(' (%0.2fx^{%0.2f})') % (exponentialCurveParams[0], exponentialCurveParams[1]), color=self.stream_settings['plot_color']) plt.plot(x, CurveFit.getYValues(CurveFit.increasingExponentialFunction, exponentialCurveParams, x), color=self.stream_settings['plot_color'], lw=2) plt.ylabel(r'$P\ (\ lag\ \leq\ TU\ )$'), plt.xlabel(getLatexForString(xlabelTimeUnits)), plt.title(getLatexForString('CDF for dimension lag distribution.')) plt.ylim((0, 1.2)) plt.legend(loc=4) if returnAxisValuesOnly: plt.show()
def plotJustifyDimensionsEstimation2(self): pltInfo = {JustifyDimensionsEstimation.top_n_dimension: {'label': 'Temporally significant', 'color': '#7109AA', 'type': '-', 'marker': 'x'}, JustifyDimensionsEstimation.first_n_dimension: {'label': 'By occurrence', 'color': '#5AF522', 'type': '-', 'marker': 'o'}} # experimentsData = {JustifyMemoryPruning.with_memory_pruning: {'iteration_time': [], 'quality': [], 'total_clusters': []}, JustifyMemoryPruning.without_memory_pruning: {'iteration_time': [], 'quality': [], 'total_clusters': []}} experimentsData = {JustifyDimensionsEstimation.top_n_dimension: defaultdict(dict), JustifyDimensionsEstimation.first_n_dimension: defaultdict(dict)} for data in FileIO.iterateJsonFromFile(JustifyDimensionsEstimation.stats_file_2): # for data in FileIO.iterateJsonFromFile('temp/dimensions_need_analysis_2'): # if 'dimensions' in data['iteration_parameters']: dimension = data['iteration_parameters']['dimensions'] type = data['iteration_parameters']['type'] if dimension not in experimentsData[type]: experimentsData[type][dimension] = {'iteration_time': [], 'quality': [], 'total_clusters': []} experimentsData[type][dimension]['iteration_time'].append(data['iteration_time']), experimentsData[type][dimension]['quality'].append(data['purity']), experimentsData[type][dimension]['total_clusters'].append(data['no_of_clusters']) lshData = dict([(k, np.mean(experimentsData[JustifyDimensionsEstimation.top_n_dimension][76819][k])) for k in experimentsData[JustifyDimensionsEstimation.top_n_dimension][76819]]) del experimentsData[JustifyDimensionsEstimation.top_n_dimension][76819] print lshData plotData = {JustifyDimensionsEstimation.top_n_dimension: defaultdict(list), JustifyDimensionsEstimation.first_n_dimension: defaultdict(list)} for type in experimentsData: for dimension in sorted(experimentsData[type]): plotData[type]['dataX'].append(dimension); [plotData[type][k].append(np.mean(experimentsData[type][dimension][k])) for k in experimentsData[type][dimension]] plt.subplot(311); for type in experimentsData: plt.semilogy([x/10**3 for x in plotData[type]['dataX']], movingAverage(plotData[type]['total_clusters'], 4), color=pltInfo[type]['color'], label=pltInfo[type]['label'], lw=2); plt.semilogy([x/10**3 for x in plotData[JustifyDimensionsEstimation.top_n_dimension]['dataX']], [lshData['total_clusters']]*len(plotData[JustifyDimensionsEstimation.top_n_dimension]['dataX']), '--', color='#FF1300', label=getLatexForString('Top-76819 dimensions'), lw=2); plt.ylim(ymin=1) plt.subplot(312); for type in experimentsData: plt.semilogy([x/10**3 for x in plotData[type]['dataX']], movingAverage(plotData[type]['iteration_time'], 4), color=pltInfo[type]['color'], label=pltInfo[type]['label'], lw=2); plt.semilogy([x/10**3 for x in plotData[JustifyDimensionsEstimation.top_n_dimension]['dataX']], [lshData['iteration_time']]*len(plotData[JustifyDimensionsEstimation.top_n_dimension]['dataX']), '--', color='#FF1300', label=getLatexForString('Top-76819'), lw=2); plt.ylim(ymin=1, ymax=1500) plt.legend(loc=2, ncol=2) plt.subplot(313); for type in experimentsData: plt.plot([x/10**3 for x in plotData[type]['dataX']], movingAverage(plotData[type]['quality'], 4), color=pltInfo[type]['color'], label=pltInfo[type]['label'], lw=2, marker=pltInfo[type]['marker']); plt.ylabel('Mean purity per iteration', fontsize=20); # plt.title(getLatexForString('Impact of dimension ranking')) plt.xlabel('# number of dimensions $(10^3)$', fontsize=20) # plt.plot([x/10**3 for x in plotData[JustifyDimensionsEstimation.top_n_dimension]['dataX']], [lshData['quality']]*len(plotData[JustifyDimensionsEstimation.top_n_dimension]['dataX']), '--', color='#FF1300', label=getLatexForString('Top-76819 dimensions'), lw=2); plt.ylim(ymin=0.80,ymax=1.0) plt.legend() plt.savefig('justifyDimensionsEstimation2.png')
def plotPercentageOfClustersWithinALag(self, returnAxisValuesOnly=True): ''' 458 Experts stream [ 0.01860266 0.70639136] 15 0.874004297177 80 Houston stream [ 0.0793181 0.47644004] 3 0.866127308876 ''' def calculatePercentageOfDecayedPhrasesFor(params, timeUnit): return 1 - CurveFit.increasingExponentialFunction(params, timeUnit) dataDistribution = {} currentTimeUnit = 0 # file='/mnt/chevron/kykamath/data/twitter/lsh_crowds/houston_stream/parameter_estimation/cluster_lag_distribution' file = self.hdsClustering.stream_settings['%s_file' % ClusteringParametersEstimation.clusterLagDistributionId] lines = list(FileIO.iterateJsonFromFile(file)) numberOfTimeUnits = len(lines) for data in lines: totalClusters = float(sum(data[ClusteringParametersEstimation.clusterLagDistributionId].values())) tempArray = [] for k, v in data[ClusteringParametersEstimation.clusterLagDistributionId].iteritems(): k = int(k) if k not in dataDistribution: dataDistribution[k] = [0] * numberOfTimeUnits dataDistribution[k][currentTimeUnit] = v / totalClusters tempArray.append(v / totalClusters) currentTimeUnit += 1 x = sorted(dataDistribution) print numberOfTimeUnits, y = getCumulativeDistribution([np.mean(dataDistribution[k]) for k in x]) params = CurveFit.getParamsAfterFittingData(x, y, CurveFit.increasingExponentialFunction, [1., 1.]) print self.stream_settings['plot_label'], params, def subPlot(id, timeUnit): plt.subplot(id) print timeUnit, calculatePercentageOfDecayedPhrasesFor(params, timeUnit) plt.plot(x, y, 'o', label=getLatexForString(self.stream_settings['plot_label']) + getLatexForString(' (%0.2fx^{%0.2f})') % (params[0], params[1]), color=self.stream_settings['plot_color']) plt.plot(x, CurveFit.getYValues(CurveFit.increasingExponentialFunction, params, x), color=self.stream_settings['plot_color'], lw=2) if self.stream_settings['stream_id'] == 'experts_twitter_stream': subPlot(111, 15); plt.title(getLatexForString('Percentage of clusters within a lag')) else: subPlot(111, 3); plt.xlabel(getLatexForString(xlabelTimeUnits)) plt.ylabel(r'$\%\ of\ clusters\ with\ lag\ \leq\ TU$') plt.legend(loc=4) if returnAxisValuesOnly: plt.show()
def plotClusteringSpeed(saveFig=True): dataToPlot = dict([(k, {'x': [], 'y': []}) for k in plotSettings]) for data in FileIO.iterateJsonFromFile(TweetsFile.stats_file): for k in plotSettings: dataToPlot[k]['x'].append(data[k]['no_of_documents']) dataToPlot[k]['y'].append(data[k]['iteration_time']) for k in plotSettings: plt.loglog(dataToPlot[k]['x'], movingAverage(dataToPlot[k]['y'], 1), label=plotSettings[k]['label'], color=plotSettings[k]['color'], lw=2) print dataToPlot['streaming_lsh']['x'][10] print dataToPlot['streaming_lsh']['y'][10] plt.legend(loc=4) if saveFig: plt.xlabel(getLatexForString('\# of documents')) plt.ylabel(getLatexForString('Running time (s)')) plt.title( getLatexForString( 'Running time comparsion for Streaing LSH with SSA')) plt.xlim(xmin=500, xmax=600000) # plt.show() if saveFig: plt.savefig('speedComparisonWithSSA.pdf')
def iteratePerformanceFrom(id): for data in FileIO.iterateJsonFromFile(getPerformanceFile(id)): del data[id]['clusters'] yield data[id]
def fileIterator(): for id in xrange(20): yield FileIO.iterateJsonFromFile(time_to_process_points + '%s' % id)
def getIterator(id): for line in FileIO.iterateJsonFromFile(time_to_process_points + 'stats/%s' % id): yield line
def kmeans(): for data in FileIO.iterateJsonFromFile( clustering_quality_experts_folder + 'combined_stats_file'): yield data['k_means']
def kmeansmr(): for data in FileIO.iterateJsonFromFile( clustering_quality_experts_folder + 'mr_quality_stats'): yield data['mr_k_means']
def unoptimized(): for data in FileIO.iterateJsonFromFile( hd_clustering_performance_folder + 'cda_unopt'): yield data['streaming_lsh']
def cdamr(): for data in FileIO.iterateJsonFromFile( clustering_quality_experts_ssa_folder + 'quality_stats'): yield data['ssa_mr']
def cdait(): for data in FileIO.iterateJsonFromFile( clustering_quality_experts_ssa_folder + 'quality_stats'): if 'ssa' in data: yield data['ssa']