def getSummarizedData(self, campaign, start, end):
collection_name = 'summarized_tweets_%s' % campaign.getId()
#print 41, datetime.now()
res = MongoManager.find(collection_name,
filters={
'start': {
"$gte": start,
"$lte": end
},
'end': {
"$lte": end
}
},
sort=('start', 1))
#pprint(res.explain())
#print 43, datetime.now()
#timerange = [SumDict(r) for r in res]
timerange = list(res)
#print 44, datetime.now()
#for r in timerange:
# print r['start'], r['end'], r['stats']['total_tweets'], r['sentiment'], r.get('calculated', '')
#print 44, datetime.now()
#print timerange[-1]['end'], end
if timerange and timerange[-1]['end'] < end:
d = self.calculateSummarizedIntervals(campaign,
timerange[-1]['end'], end,
end - timerange[-1]['end'])
#for k in d:
# k['calculated'] = True
timerange.extend(d)
#for r in timerange:
# print r['start'], r['end'], r['stats']['total_tweets'], r['sentiment'], r.get('calculated', '')
#print 45, datetime.now()
return timerange
def getNamesDatabase(cls, **kwargs):
max_age = kwargs.get('max_age', timedelta(seconds=0))
if not max_age or not cls.cached_names_database or (datetime.now() - cls.cached_names_database['fetch_time'] > max_age):
namesdb = MongoManager.find("gender_names")
res = {}
for name in namesdb:
res[name["name"].lower()] = name["gender"]
cls.cached_names_database = {'data': res, 'fetch_time': datetime.now()}
return cls.cached_names_database['data']
def getLastSummarizedDate(self, campaign):
collection_name = 'summarized_tweets_%s' % campaign.getId()
res = MongoManager.find(collection_name, sort=("start", -1), limit=1)
if res.count():
return res[0]['end']
else:
collection_name = 'tweets_%s' % campaign.getId()
res = MongoManager.findTweets(collection_name, sort=("x_created_at", 1), limit=1)
if res.count():
return res[0]['x_created_at'].replace(minute=0, second=0, microsecond=0)
return datetime.now().replace(minute=0, second=0, microsecond=0)
def getNamesDatabase(cls, **kwargs):
max_age = kwargs.get('max_age', timedelta(seconds=0))
if not max_age or not cls.cached_names_database or (
datetime.now() - cls.cached_names_database['fetch_time'] >
max_age):
namesdb = MongoManager.find("gender_names")
res = {}
for name in namesdb:
res[name["name"].lower()] = name["gender"]
cls.cached_names_database = {
'data': res,
'fetch_time': datetime.now()
}
return cls.cached_names_database['data']
def getLastSummarizedDate(self, campaign):
collection_name = 'summarized_tweets_%s' % campaign.getId()
res = MongoManager.find(collection_name, sort=("start", -1), limit=1)
if res.count():
return res[0]['end']
else:
collection_name = 'tweets_%s' % campaign.getId()
res = MongoManager.findTweets(collection_name,
sort=("x_created_at", 1),
limit=1)
if res.count():
return res[0]['x_created_at'].replace(minute=0,
second=0,
microsecond=0)
return datetime.now().replace(minute=0, second=0, microsecond=0)
def getSummarizedData(self, campaign, start, end):
collection_name = 'summarized_tweets_%s' % campaign.getId()
#print 41, datetime.now()
res = MongoManager.find(collection_name, filters={'start': {"$gte": start, "$lte": end}, 'end': {"$lte": end}}, sort=('start',1))
#pprint(res.explain())
#print 43, datetime.now()
#timerange = [SumDict(r) for r in res]
timerange = list(res)
#print 44, datetime.now()
#for r in timerange:
# print r['start'], r['end'], r['stats']['total_tweets'], r['sentiment'], r.get('calculated', '')
#print 44, datetime.now()
#print timerange[-1]['end'], end
if timerange and timerange[-1]['end'] < end:
d = self.calculateSummarizedIntervals(campaign, timerange[-1]['end'], end, end - timerange[-1]['end'])
#for k in d:
# k['calculated'] = True
timerange.extend(d)
#for r in timerange:
# print r['start'], r['end'], r['stats']['total_tweets'], r['sentiment'], r.get('calculated', '')
#print 45, datetime.now()
return timerange
