def getSummarizedData(self, campaign, start, end): collection_name = 'summarized_tweets_%s' % campaign.getId() #print 41, datetime.now() res = MongoManager.find(collection_name, filters={ 'start': { "$gte": start, "$lte": end }, 'end': { "$lte": end } }, sort=('start', 1)) #pprint(res.explain()) #print 43, datetime.now() #timerange = [SumDict(r) for r in res] timerange = list(res) #print 44, datetime.now() #for r in timerange: # print r['start'], r['end'], r['stats']['total_tweets'], r['sentiment'], r.get('calculated', '') #print 44, datetime.now() #print timerange[-1]['end'], end if timerange and timerange[-1]['end'] < end: d = self.calculateSummarizedIntervals(campaign, timerange[-1]['end'], end, end - timerange[-1]['end']) #for k in d: # k['calculated'] = True timerange.extend(d) #for r in timerange: # print r['start'], r['end'], r['stats']['total_tweets'], r['sentiment'], r.get('calculated', '') #print 45, datetime.now() return timerange
def getNamesDatabase(cls, **kwargs): max_age = kwargs.get('max_age', timedelta(seconds=0)) if not max_age or not cls.cached_names_database or (datetime.now() - cls.cached_names_database['fetch_time'] > max_age): namesdb = MongoManager.find("gender_names") res = {} for name in namesdb: res[name["name"].lower()] = name["gender"] cls.cached_names_database = {'data': res, 'fetch_time': datetime.now()} return cls.cached_names_database['data']
def getLastSummarizedDate(self, campaign): collection_name = 'summarized_tweets_%s' % campaign.getId() res = MongoManager.find(collection_name, sort=("start", -1), limit=1) if res.count(): return res[0]['end'] else: collection_name = 'tweets_%s' % campaign.getId() res = MongoManager.findTweets(collection_name, sort=("x_created_at", 1), limit=1) if res.count(): return res[0]['x_created_at'].replace(minute=0, second=0, microsecond=0) return datetime.now().replace(minute=0, second=0, microsecond=0)
def getNamesDatabase(cls, **kwargs): max_age = kwargs.get('max_age', timedelta(seconds=0)) if not max_age or not cls.cached_names_database or ( datetime.now() - cls.cached_names_database['fetch_time'] > max_age): namesdb = MongoManager.find("gender_names") res = {} for name in namesdb: res[name["name"].lower()] = name["gender"] cls.cached_names_database = { 'data': res, 'fetch_time': datetime.now() } return cls.cached_names_database['data']
def getSummarizedData(self, campaign, start, end): collection_name = 'summarized_tweets_%s' % campaign.getId() #print 41, datetime.now() res = MongoManager.find(collection_name, filters={'start': {"$gte": start, "$lte": end}, 'end': {"$lte": end}}, sort=('start',1)) #pprint(res.explain()) #print 43, datetime.now() #timerange = [SumDict(r) for r in res] timerange = list(res) #print 44, datetime.now() #for r in timerange: # print r['start'], r['end'], r['stats']['total_tweets'], r['sentiment'], r.get('calculated', '') #print 44, datetime.now() #print timerange[-1]['end'], end if timerange and timerange[-1]['end'] < end: d = self.calculateSummarizedIntervals(campaign, timerange[-1]['end'], end, end - timerange[-1]['end']) #for k in d: # k['calculated'] = True timerange.extend(d) #for r in timerange: # print r['start'], r['end'], r['stats']['total_tweets'], r['sentiment'], r.get('calculated', '') #print 45, datetime.now() return timerange