def dump_data(self):
if not self.public_url:
returnValue(False)
stats = yield find_stats({'user': self.user}, filter=sortasc('timestamp'), timeout=120)
dates = [s['timestamp'] for s in stats]
tweets = [s['tweets'] for s in stats]
tweets_diff = [a - b for a, b in zip(tweets[1:],tweets[:-1])]
followers = [s['followers'] for s in stats]
followers_diff = [a - b for a, b in zip(followers[1:], followers[:-1])]
rts_diff = [s['rts_last_hour'] for s in stats]
rts = []
n = 0
for a in rts_diff:
n += a
rts.append(n)
jsondata = {}
imax = len(dates) - 1
for i, date in enumerate(dates):
ts = int(time.mktime(date.timetuple()))
jsondata[ts] = { 'tweets': tweets[i], 'followers': followers[i], 'rts': rts[i] }
if i < imax:
jsondata[ts].update({ 'tweets_diff': tweets_diff[i], 'followers_diff': followers_diff[i], 'rts_diff': rts_diff[i+1] })
try:
jsondir = os.path.join('web', 'data')
if not os.path.exists(jsondir):
os.makedirs(jsondir)
os.chmod(jsondir, 0o755)
with open(os.path.join(jsondir, 'stats_%s.json' % self.user), 'w') as outfile:
write_json(jsondata, outfile)
except IOError as e:
loggerr("Could not write web/data/stats_%s.json : %s" % (self.user, e), action="stats")
try:
from plots import CumulativeCurve, DailyHistogram, WeekPunchCard
imgdir = os.path.join('web', 'img')
if not os.path.exists(imgdir):
os.makedirs(imgdir)
os.chmod(imgdir, 0o755)
CumulativeCurve(dates, tweets, 'Total tweets', imgdir, 'tweets_%s' % self.user)
CumulativeCurve(dates, followers, 'Total followers', imgdir, 'followers_%s' % self.user)
CumulativeCurve(dates, rts, 'Total RTs since %s' % dates[0], imgdir, 'rts_%s' % self.user)
DailyHistogram(dates[:-1], tweets_diff, 'New tweets', imgdir, 'new_tweets_%s' % self.user)
DailyHistogram(dates[:-1], followers_diff, 'New followers', imgdir, 'new_followers_%s' % self.user)
DailyHistogram(dates[:-1], rts_diff[1:], 'New RTs', imgdir, 'new_rts_%s' % self.user)
WeekPunchCard(dates[:-1], tweets_diff, 'Tweets punchcard', imgdir, 'tweets_card_%s' % self.user)
WeekPunchCard(dates[:-1], followers_diff, 'Followers punchcard', imgdir, 'followers_card_%s' % self.user)
WeekPunchCard(dates[:-1], rts_diff[1:], 'RTs punchcard', imgdir, 'rts_card_%s' % self.user)
except Exception as e:
loggerr("Could not write images in web/img for %s : %s" % (self.user, e), action="stats")
data = {'user': self.user, 'url': self.public_url}
self.render_template("static_stats.html", self.user, data)
returnValue(True)
def print_last(self):
now = timestamp_hour(datetime.today())
since = now - timedelta(days=30)
stats = yield find_stats({'user': self.user, 'timestamp': {'$gte': since}}, filter=sortdesc('timestamp'))
if not len(stats):
returnValue()
stat = stats[0]
stat["followers"] = yield count_followers(self.user)
rts = 0
fols = 0
twts = 0
delays = {1: 'hour', 6: '6 hours', 24: 'day', 7*24: 'week', 30*24: 'month'}
order = delays.keys()
order.sort()
olds = {'tweets': {}, 'followers': {}, 'rts': {}}
for s in stats:
d = now - s['timestamp']
delay = d.seconds / 3600 + d.days * 24
fols = stat['followers'] - s['followers']
twts = stat['tweets'] - s['tweets']
for i in order:
if delay == i:
if 'stats%sH' % i not in olds['tweets']:
olds['tweets']['stats%sH' % i] = twts if twts not in olds['tweets'].values() else 0
if 'stats%sH' % i not in olds['followers']:
olds['followers']['stats%sH' % i] = fols if fols not in olds['followers'].values() else 0
if 'stats%sH' % i not in olds['rts']:
olds['rts']['stats%sH' % i] = rts if rts not in olds['rts'].values() else 0
rts += s['rts_last_hour']
olds['rts']['stats1H'] = stat['rts_last_hour']
for i in order:
if rts and 'stats%sH' % i not in olds['rts'] and rts not in olds['rts'].values():
olds['rts']['stats%sH' % i] = rts
rts = 0
if fols and 'stats%sH' % i not in olds['followers'] and fols not in olds['followers'].values():
olds['followers']['stats%sH' % i] = fols
fols = 0
if twts and 'stats%sH' % i not in olds['tweets'] and twts not in olds['tweets'].values():
olds['tweets']['stats%sH' % i] = twts
twts = 0
res = []
if stat['tweets']:
res.append("Tweets: %d total" % stat['tweets'] + " ; ".join([""]+["%d last %s" % (olds['tweets']['stats%sH' % i], delays[i]) for i in order if 'stats%sH' % i in olds['tweets'] and olds['tweets']['stats%sH' % i]]))
textrts = ["%d last %s" % (olds['rts']['stats%sH' % i], delays[i]) for i in order if 'stats%sH' % i in olds['rts'] and olds['rts']['stats%sH' % i]]
if textrts:
res.append("RTs: " + " ; ".join(textrts))
if stat['followers']:
res.append("Followers: %d total" % stat['followers'] + " ; ".join([""]+["%+d last %s" % (olds['followers']['stats%sH' % i], delays[i]) for i in order if 'stats%sH' % i in olds['followers'] and olds['followers']['stats%sH' % i]]))
recent = yield find_last_followers(self.user)
if recent:
res.append("Recent follower%s: %s" % ("s include" if len(recent) > 1 else "", format_4_followers(recent)))
if self.public_url and res:
res.append("More details: %sstatic_stats_%s.html" % (self.public_url, self.user))
returnValue([(True, "[Stats] %s" % m) for m in res])
def get_stats(self, **kwargs):
timestamp = timestamp_hour(datetime.today())
try:
last = yield find_stats({'user': self.user.lower()}, limit=1, filter=sortdesc('timestamp'))
last = last[0]
except:
last = {}
if last and last['timestamp'] == timestamp:
res = None
else:
res = self._send_query(self.conn.users.show, {'screen_name': self.user}, return_result=True)
check_twitter_results(res)
returnValue((res, last, timestamp))
def get_stats(self, **kwargs):
timestamp = timestamp_hour(datetime.today())
try:
last = yield find_stats({'user': self.user.lower()},
limit=1,
filter=sortdesc('timestamp'))
last = last[0]
except:
last = {}
if last and last['timestamp'] == timestamp:
res = None
else:
res = self._send_query(self.conn.users.show,
{'screen_name': self.user},
return_result=True)
check_twitter_results(res)
returnValue((res, last, timestamp))
