def process_dms(self, listdms, user):
if not listdms:
returnD(False)
ids = []
dms = []
if not isinstance(listdms, list):
self.log("downloading DMs: %s" % listdms, error=True)
returnD(False)
for i in listdms:
try:
#date = datetime.strptime(i.get('created_at', ''), '%a %b %d %H:%M:%S +0000 %Y') + timedelta(hours=2)
date = parse_date(i.get('created_at', ''))
if datetime.today() - date > timedelta(
hours=config.BACK_HOURS):
break
except Exception as e:
self.log("processing DM %s: %s %s" %
(i.get('created_at'), type(e), e),
error=True)
continue
tid = long(i.get('id', ''))
if tid:
ids.append(tid)
sender = i.get('sender_screen_name', '')
dm, self.fact.cache_urls = yield clean_redir_urls(
i.get('text', '').replace('\n', ' '), self.fact.cache_urls)
dms.append({
'_id': "%s:%s" % (self.fact.channel, tid),
'channel': self.fact.channel,
'id': tid,
'user': user,
'sender': sender.lower(),
'screenname': sender,
'message': dm,
'date': date,
'timestamp': datetime.today()
})
existings = yield self.fact.db['dms'].find(
{
'channel': self.fact.channel,
'id': {
'$in': ids
}
},
fields=['_id'],
filter=sortdesc('id'))
existing = [t['_id'] for t in existings]
news = [t for t in dms if t['_id'] not in existing]
if news:
news.reverse()
yield self.fact.db['dms'].insert(news, safe=True)
self.fact.ircclient._send_message([(
True, "[DM] @%s: %s — https://twitter.com/%s" %
(n['screenname'].encode('utf-8'), n['message'].encode('utf-8'),
n['screenname'].encode('utf-8'))) for n in news],
self.fact.channel)
returnD(True)
def get_stats(self, **kwargs):
timestamp = timestamp_hour(datetime.today())
try:
last = yield find_stats({'user': self.user.lower()}, limit=1, filter=sortdesc('timestamp'))
last = last[0]
except:
last = {}
if last and last['timestamp'] == timestamp:
res = None
else:
res = self._send_query(self.conn.users.show, {'screen_name': self.user}, return_result=True)
check_twitter_results(res)
returnValue((res, last, timestamp))
def get_stats(self, **kwargs):
timestamp = timestamp_hour(datetime.today())
try:
last = yield find_stats({'user': self.user.lower()},
limit=1,
filter=sortdesc('timestamp'))
last = last[0]
except:
last = {}
if last and last['timestamp'] == timestamp:
res = None
else:
res = self._send_query(self.conn.users.show,
{'screen_name': self.user},
return_result=True)
check_twitter_results(res)
returnValue((res, last, timestamp))
def print_last(self):
now = timestamp_hour(datetime.today())
since = now - timedelta(days=30)
stats = yield find_stats({'user': self.user, 'timestamp': {'$gte': since}}, filter=sortdesc('timestamp'))
if not len(stats):
returnValue()
stat = stats[0]
stat["followers"] = yield count_followers(self.user)
rts = 0
fols = 0
twts = 0
delays = {1: 'hour', 6: '6 hours', 24: 'day', 7*24: 'week', 30*24: 'month'}
order = delays.keys()
order.sort()
olds = {'tweets': {}, 'followers': {}, 'rts': {}}
for s in stats:
d = now - s['timestamp']
delay = d.seconds / 3600 + d.days * 24
fols = stat['followers'] - s['followers']
twts = stat['tweets'] - s['tweets']
for i in order:
if delay == i:
if 'stats%sH' % i not in olds['tweets']:
olds['tweets']['stats%sH' % i] = twts if twts not in olds['tweets'].values() else 0
if 'stats%sH' % i not in olds['followers']:
olds['followers']['stats%sH' % i] = fols if fols not in olds['followers'].values() else 0
if 'stats%sH' % i not in olds['rts']:
olds['rts']['stats%sH' % i] = rts if rts not in olds['rts'].values() else 0
rts += s['rts_last_hour']
olds['rts']['stats1H'] = stat['rts_last_hour']
for i in order:
if rts and 'stats%sH' % i not in olds['rts'] and rts not in olds['rts'].values():
olds['rts']['stats%sH' % i] = rts
rts = 0
if fols and 'stats%sH' % i not in olds['followers'] and fols not in olds['followers'].values():
olds['followers']['stats%sH' % i] = fols
fols = 0
if twts and 'stats%sH' % i not in olds['tweets'] and twts not in olds['tweets'].values():
olds['tweets']['stats%sH' % i] = twts
twts = 0
res = []
if stat['tweets']:
res.append("Tweets: %d total" % stat['tweets'] + " ; ".join([""]+["%d last %s" % (olds['tweets']['stats%sH' % i], delays[i]) for i in order if 'stats%sH' % i in olds['tweets'] and olds['tweets']['stats%sH' % i]]))
textrts = ["%d last %s" % (olds['rts']['stats%sH' % i], delays[i]) for i in order if 'stats%sH' % i in olds['rts'] and olds['rts']['stats%sH' % i]]
if textrts:
res.append("RTs: " + " ; ".join(textrts))
if stat['followers']:
res.append("Followers: %d total" % stat['followers'] + " ; ".join([""]+["%+d last %s" % (olds['followers']['stats%sH' % i], delays[i]) for i in order if 'stats%sH' % i in olds['followers'] and olds['followers']['stats%sH' % i]]))
recent = yield find_last_followers(self.user)
if recent:
res.append("Recent follower%s: %s" % ("s include" if len(recent) > 1 else "", format_4_followers(recent)))
if self.public_url and res:
res.append("More details: %sstatic_stats_%s.html" % (self.public_url, self.user))
returnValue([(True, "[Stats] %s" % m) for m in res])
def process_stats(self, stats, user):
# Update followers list
conf = chanconf(self.fact.channel)
conn = Microblog('twitter', conf, bearer_token=conf["oauth2"])
lost = yield conn.update_followers(self.fact.db)
ct = len(lost)
if ct:
self.fact.ircclient._send_message('[twitter] Lost %s follower%s: %s%s' % (ct, "s" if ct>1 else "", format_4_followers(lost), "…" if ct>4 else ""), self.fact.channel)
# Update stats
if not stats:
returnD(False)
stats, last, timestamp = stats
if not stats or type(stats) is str:
returnD(False)
if not last:
last = {'tweets': 0, 'followers': 0}
since = timestamp - timedelta(hours=1)
else:
since = last['timestamp']
if 'lists' not in last:
last['lists'] = 0
re_match_rts = re.compile(u'(([MLR]T|%s|♺)\s*)+@?%s' % (QUOTE_CHARS, user), re.I)
rts = yield self.fact.db['tweets'].find({'channel': self.fact.channel, 'message': re_match_rts, 'timestamp': {'$gte': since}}, fields=['_id'])
nb_rts = len(rts)
nb_fols = yield count_followers(user)
stat = {'user': user, 'timestamp': timestamp, 'tweets': stats.get('statuses_count', last['tweets']), 'followers': nb_fols, 'rts_last_hour': nb_rts, 'lists': stats.get('listed_count', last['lists'])}
yield self.fact.db['stats'].insert(stat)
weekday = timestamp.weekday()
laststats = Stats(user)
if chan_displays_stats(self.fact.channel) and ((timestamp.hour == 13 and weekday < 5) or timestamp.hour == 18):
stats = yield laststats.print_last()
self.fact.ircclient._send_message(stats, self.fact.channel)
last_tweet = yield self.fact.db['tweets'].find({'channel': self.fact.channel, 'user': user}, fields=['date'], limit=1, filter=sortdesc('timestamp'))
if chan_displays_stats(self.fact.channel) and last_tweet and timestamp - last_tweet[0]['date'] > timedelta(days=3) and (timestamp.hour == 11 or timestamp.hour == 17) and weekday < 5:
reactor.callFromThread(reactor.callLater, 3, self.fact.ircclient._send_message, "[FYI] No tweet was sent since %s days." % (timestamp - last_tweet[0]['date']).days, self.fact.channel)
reactor.callFromThread(reactor.callLater, 1, laststats.dump_data)
returnD(True)
def process_dms(self, listdms, user):
if not listdms:
returnD(False)
ids = []
dms = []
if not isinstance(listdms, list):
self.log("downloading DMs: %s" % listdms, error=True)
returnD(False)
for i in listdms:
try:
date = datetime.fromtimestamp(time.mktime(time.strptime(i.get('created_at', ''), '%a %b %d %H:%M:%S +0000 %Y'))+2*60*60)
if datetime.today() - date > timedelta(hours=config.BACK_HOURS):
break
except:
self.log("processing DM %s: %s" % (i, listdms), error=True)
continue
tid = long(i.get('id', ''))
if tid:
ids.append(tid)
sender = i.get('sender_screen_name', '')
dm, self.fact.cache_urls = yield clean_redir_urls(i.get('text', '').replace('\n', ' '), self.fact.cache_urls)
dms.append({'_id': "%s:%s" % (self.fact.channel, tid), 'channel': self.fact.channel, 'id': tid, 'user': user, 'sender': sender.lower(), 'screenname': sender, 'message': dm, 'date': date, 'timestamp': datetime.today()})
existings = yield self.fact.db['dms'].find({'channel': self.fact.channel, 'id': {'$in': ids}}, fields=['_id'], filter=sortdesc('id'))
existing = [t['_id'] for t in existings]
news = [t for t in dms if t['_id'] not in existing]
if news:
news.reverse()
yield self.fact.db['dms'].insert(news, safe=True)
self.fact.ircclient._send_message([(True, "[DM] @%s: %s — https://twitter.com/%s" % (n['screenname'].encode('utf-8'), n['message'].encode('utf-8'), n['screenname'].encode('utf-8'))) for n in news], self.fact.channel)
returnD(True)
def process_tweets(self, feed, source, query=None, pagecount=0):
# handle tweets from icerocket or topsy fake rss
nexturl = ""
try:
elements = feed.entries
except:
# handle tweets from Twitter API
if isinstance(feed, list) and len(feed):
elements = feed
elif isinstance(feed, dict) and "nexturl" in feed:
nexturl = feed["nexturl"]
elements = feed["tweets"]
else:
returnD(False)
if query:
source = "%s https://api.twitter.com/api/1.1/search/tweets.json?q=%s" % (source, query)
ids = []
hashs = []
tweets = []
fresh = True
for i in elements:
try:
time_tweet = time.mktime(i.get('published_parsed', '')) - 4*60*60
except:
if i.get('created_at', '') == "now":
time_tweet = time.time()
else:
time_tweet = time.mktime(time.strptime(i.get('created_at', ''), '%a %b %d %H:%M:%S +0000 %Y')) + 2*60*60
date = datetime.fromtimestamp(time_tweet)
if datetime.today() - date > timedelta(hours=config.BACK_HOURS):
fresh = False
break
tweet, self.fact.cache_urls = yield clean_redir_urls(i.get('title', '').replace('\n', ' '), self.fact.cache_urls)
link = i.get('link', '')
res = re_tweet_url.search(link)
if res:
user = res.group(1)
tid = long(res.group(2))
ids.append(tid)
tw = {'_id': "%s:%s" % (self.fact.channel, tid), 'channel': self.fact.channel, 'id': tid, 'user': user.lower(), 'screenname': user, 'message': tweet, 'uniq_rt_hash': uniq_rt_hash(tweet), 'link': link, 'date': date, 'timestamp': datetime.today(), 'source': source}
tw = grab_extra_meta(i, tw)
tweets.append(tw)
# Delay displaying to avoid duplicates from the stream
if source != "mystream" and not self.fact.tweets_search_page:
yield deferredSleep()
existings = yield self.fact.db['tweets'].find({'channel': self.fact.channel, 'id': {'$in': ids}}, fields=['_id'], filter=sortdesc('id'))
existing = [t['_id'] for t in existings]
news = [t for t in tweets if t['_id'] not in existing]
if not news:
returnD(False)
good = []
news.sort(key=itemgetter('id'))
if fresh and not source.startswith("my") and len(news) > len(elements) / 2:
if query and nexturl and pagecount < 3*self.fact.back_pages_limit:
deferToThreadPool(reactor, self.threadpool, reactor.callLater, 15, self.start_twitter_search, [query], max_id=nexturl, pagecount=pagecount+1)
elif not query and nexturl and "p=%d" % (self.fact.back_pages_limit+1) not in nexturl and "page=%s" % (2*self.fact.back_pages_limit) not in nexturl:
deferToThreadPool(reactor, self.threadpool, reactor.callLater, 41, self.start, nexturl)
elif not query and not nexturl and int(source[-1:]) <= self.fact.back_pages_limit:
deferToThreadPool(reactor, self.threadpool, reactor.callLater, 41, self.start, next_page(source))
if self.fact.displayRT:
good = news
else:
hashs = [t['uniq_rt_hash'] for t in news if t['uniq_rt_hash'] not in hashs]
existings = yield self.fact.db['tweets'].find({'channel': self.fact.channel, 'uniq_rt_hash': {'$in': hashs}}, fields=['uniq_rt_hash'], filter=sortdesc('id'))
existing = [t['uniq_rt_hash'] for t in existings]
for t in news:
if self.fact.twuser == t['user'] or t['uniq_rt_hash'] not in existing or (self.fact.displayMyRT and "@%s" % self.fact.twuser in t['message'].lower()):
existing.append(t['uniq_rt_hash'])
good.append(t)
if config.DEBUG:
nb_rts_str = ""
nb_rts = len(news) - len(good)
if nb_rts:
nb_rts_str = " (%s RTs filtered)" % nb_rts
self.log("Displaying %s tweets%s" % (len(good), nb_rts_str), hint=True)
if self.fact.status != "closed":
for t in good:
msg = "%s: %s" % (t['screenname'].encode('utf-8'), self.format_tweet(t))
self.fact.ircclient._send_message(msg, self.fact.channel)
for t in news:
yield self.fact.db['tweets'].save(t, safe=True)
returnD(True)
def process_elements(self, feed, url):
if not feed or not feed.entries:
returnD(False)
sourcename = url
if feed.feed and 'title' in feed.feed:
sourcename = feed.feed['title']
sourcename = unescape_html(sourcename)
ids = []
news = []
links = []
for i in feed.entries:
date = i.get('published_parsed', i.get('updated_parsed', ''))
if date:
date = datetime.fromtimestamp(time.mktime(date))
if datetime.today() - date > timedelta(hours=config.BACK_HOURS+6):
break
link, self.fact.cache_urls = yield clean_redir_urls(i.get('link', ''), self.fact.cache_urls)
if not link.startswith('http'):
link = "%s/%s" % (url[:url.find('/',8)], link.lstrip('/'))
if link in links:
continue
links.append(link)
title = i.get('title', '').replace('\n', ' ')
try:
title = unescape_html(title)
except:
pass
_id = md5(("%s:%s:%s" % (self.fact.channel, link, title.lower())).encode('utf-8')).hexdigest()
ids.append(_id)
news.append({'_id': _id, 'channel': self.fact.channel, 'message': title, 'link': link, 'date': date, 'timestamp': datetime.today(), 'source': url, 'sourcename': sourcename})
existings = yield self.fact.db['news'].find({'channel': self.fact.channel, '_id': {'$in': ids}}, fields=['_id'], filter=sortdesc('_id'))
existing = [n['_id'] for n in existings]
new = [n for n in news if n['_id'] not in existing]
if new:
new.reverse()
new = new[:5]
try:
yield self.fact.db['news'].insert(new, safe=True)
except Exception as e:
self._handle_error(e, "recording news batch", url)
self.fact.ircclient._send_message([(True, "[%s] %s" % (n['sourcename'].encode('utf-8'), self.format_tweet(n))) for n in new], self.fact.channel)
returnD(True)
def process_stats(self, res, user):
if not res:
returnD(False)
stats, last, timestamp = res
if not stats:
returnD(False)
if not last:
last = {'tweets': 0, 'followers': 0}
since = timestamp - timedelta(hours=1)
else:
since = last['timestamp']
if 'lists' not in last:
last['lists'] = 0
re_match_rts = re.compile(u'(([MLR]T|%s|♺)\s*)+@?%s' % (QUOTE_CHARS, user), re.I)
rts = yield Mongo('tweets', 'find', {'channel': self.fact.channel, 'message': re_match_rts, 'timestamp': {'$gte': since}}, fields=['_id'])
nb_rts = len(rts)
if config.TWITTER_API_VERSION == 1:
stat = {'user': user, 'timestamp': timestamp, 'tweets': stats.get('updates', last['tweets']), 'followers': stats.get('followers', last['followers']), 'rts_last_hour': nb_rts}
else:
stat = {'user': user, 'timestamp': timestamp, 'tweets': stats.get('statuses_count', last['tweets']), 'followers': stats.get('followers_count', last['followers']), 'rts_last_hour': nb_rts, 'lists': stats.get('listed_count', last['lists'])}
yield Mongo('stats', 'insert', stat)
weekday = timestamp.weekday()
laststats = Stats(user)
if chan_displays_stats(self.fact.channel) and ((timestamp.hour == 13 and weekday < 5) or timestamp.hour == 18):
stats = yield laststats.print_last()
self.fact.ircclient._send_message(stats, self.fact.channel)
last_tweet = yield Mongo('tweets', 'find', {'channel': self.fact.channel, 'user': user}, fields=['date'], limit=1, filter=sortdesc('timestamp'))
if chan_displays_stats(self.fact.channel) and last_tweet and timestamp - last_tweet[0]['date'] > timedelta(days=3) and (timestamp.hour == 11 or timestamp.hour == 17) and weekday < 5:
reactor.callFromThread(reactor.callLater, 3, self.fact.ircclient._send_message, "[FYI] No tweet was sent since %s days." % (timestamp - last_tweet[0]['date']).days, self.fact.channel)
reactor.callFromThread(reactor.callLater, 1, laststats.dump_data)
returnD(True)
def process_stats(self, stats, user):
# Update followers list
conf = chanconf(self.fact.channel)
conn = Microblog('twitter', conf, bearer_token=conf["oauth2"])
lost = yield conn.update_followers(self.fact.db)
ct = len(lost)
if ct:
self.fact.ircclient._send_message(
'[twitter] Lost %s follower%s: %s%s' %
(ct, "s" if ct > 1 else "", format_4_followers(lost),
"…" if ct > 4 else ""), self.fact.channel)
# Update stats
if not stats:
returnD(False)
stats, last, timestamp = stats
if not stats or type(stats) is str:
returnD(False)
if not last:
last = {'tweets': 0, 'followers': 0}
since = timestamp - timedelta(hours=1)
else:
since = last['timestamp']
if 'lists' not in last:
last['lists'] = 0
re_match_rts = re.compile(
u'(([MLR]T|%s|♺)\s*)+@?%s' % (QUOTE_CHARS, user), re.I)
rts = yield self.fact.db['tweets'].find(
{
'channel': self.fact.channel,
'message': re_match_rts,
'timestamp': {
'$gte': since
}
},
fields=['_id'])
nb_rts = len(rts)
nb_fols = yield count_followers(user)
stat = {
'user': user,
'timestamp': timestamp,
'tweets': stats.get('statuses_count', last['tweets']),
'followers': nb_fols,
'rts_last_hour': nb_rts,
'lists': stats.get('listed_count', last['lists'])
}
yield self.fact.db['stats'].insert(stat)
weekday = timestamp.weekday()
laststats = Stats(user)
if chan_displays_stats(self.fact.channel) and (
(timestamp.hour == 13 and weekday < 5) or timestamp.hour == 18):
stats = yield laststats.print_last()
self.fact.ircclient._send_message(stats, self.fact.channel)
last_tweet = yield self.fact.db['tweets'].find(
{
'channel': self.fact.channel,
'user': user
},
fields=['date'],
limit=1,
filter=sortdesc('timestamp'))
if chan_displays_stats(
self.fact.channel
) and last_tweet and timestamp - last_tweet[0]['date'] > timedelta(
days=3) and (timestamp.hour == 11
or timestamp.hour == 17) and weekday < 5:
reactor.callFromThread(
reactor.callLater, 3, self.fact.ircclient._send_message,
"[FYI] No tweet was sent since %s days." %
(timestamp - last_tweet[0]['date']).days, self.fact.channel)
reactor.callFromThread(reactor.callLater, 1, laststats.dump_data)
returnD(True)
def process_tweets(self, feed, source, query=None, pagecount=0):
# handle tweets from icerocket or topsy fake rss
nexturl = ""
try:
elements = feed.entries
except:
# handle tweets from Twitter API
if isinstance(feed, list) and len(feed):
elements = feed
elif isinstance(feed, dict) and "nexturl" in feed:
nexturl = feed["nexturl"]
elements = feed["tweets"]
else:
returnD(False)
if query:
source = "%s https://api.twitter.com/api/1.1/search/tweets.json?q=%s" % (
source, query)
ids = []
hashs = []
tweets = []
fresh = True
for i in elements:
try:
date = datetime.fromtimestamp(
time.mktime(i.get('published_parsed', '')) - 4 * 60 * 60)
except:
if i.get('created_at', '') == "now":
date = datetime.now()
else:
#date = datetime.strptime(i.get('created_at', ''), '%a %b %d %H:%M:%S +0000 %Y') + timedelta(hours=2)
date = parse_date(i.get('created_at', ''))
if datetime.today() - date > timedelta(hours=config.BACK_HOURS):
fresh = False
break
tweet, self.fact.cache_urls = yield clean_redir_urls(
i.get('title', '').replace('\n', ' '), self.fact.cache_urls)
link = i.get('link', '')
res = re_tweet_url.search(link)
if res:
user = res.group(1)
tid = long(res.group(2))
ids.append(tid)
tw = {
'_id': "%s:%s" % (self.fact.channel, tid),
'channel': self.fact.channel,
'id': tid,
'user': user.lower(),
'screenname': user,
'message': tweet,
'uniq_rt_hash': uniq_rt_hash(tweet),
'link': link,
'date': date,
'timestamp': datetime.today(),
'source': source
}
tw = grab_extra_meta(i, tw)
tweets.append(tw)
# Delay displaying to avoid duplicates from the stream
if source != "mystream" and not self.fact.tweets_search_page:
yield deferredSleep()
existings = yield self.fact.db['tweets'].find(
{
'channel': self.fact.channel,
'id': {
'$in': ids
}
},
fields=['_id'],
filter=sortdesc('id'))
existing = [t['_id'] for t in existings]
news = [t for t in tweets if t['_id'] not in existing]
if not news:
returnD(False)
good = []
news.sort(key=itemgetter('id'))
if fresh and not source.startswith(
"my") and len(news) > len(elements) / 2:
if query and nexturl and pagecount < 3 * self.fact.back_pages_limit:
deferToThreadPool(reactor,
self.threadpool,
reactor.callLater,
15,
self.start_twitter_search, [query],
max_id=nexturl,
pagecount=pagecount + 1)
elif not query and nexturl and "p=%d" % (
self.fact.back_pages_limit +
1) not in nexturl and "page=%s" % (
2 * self.fact.back_pages_limit) not in nexturl:
deferToThreadPool(reactor, self.threadpool, reactor.callLater,
41, self.start_web, nexturl)
elif not query and not nexturl and int(
source[-1:]) <= self.fact.back_pages_limit:
deferToThreadPool(reactor, self.threadpool, reactor.callLater,
41, self.start_web, next_page(source))
if self.fact.displayRT:
good = news
else:
hashs = [
t['uniq_rt_hash'] for t in news
if t['uniq_rt_hash'] not in hashs
]
existings = yield self.fact.db['tweets'].find(
{
'channel': self.fact.channel,
'uniq_rt_hash': {
'$in': hashs
}
},
fields=['uniq_rt_hash'],
filter=sortdesc('id'))
existing = [t['uniq_rt_hash'] for t in existings]
for t in news:
if self.fact.twuser == t['user'] or t[
'uniq_rt_hash'] not in existing or (
self.fact.displayMyRT and "@%s" % self.fact.twuser
in t['message'].lower()):
existing.append(t['uniq_rt_hash'])
good.append(t)
if config.DEBUG:
nb_rts_str = ""
nb_rts = len(news) - len(good)
if nb_rts:
nb_rts_str = " (%s RTs filtered)" % nb_rts
self.log("Displaying %s tweets%s" % (len(good), nb_rts_str),
hint=True)
if self.fact.status != "closed":
for t in good:
msg = "%s: %s" % (t['screenname'].encode('utf-8'),
self.format_tweet(t))
self.fact.ircclient._send_message(msg, self.fact.channel)
for t in news:
yield self.fact.db['tweets'].save(t, safe=True)
returnD(True)
def process_elements(self, data, url, name=None):
if not data:
returnD(False)
if self.fact.name == "pages":
differ = WebMonitor(name, url, self.fact.channel)
info = yield differ.check_new(data)
if info:
self.fact.ircclient._send_message(info, self.fact.channel)
returnD(True)
if not data.entries:
returnD(False)
sourcename = url
if data.feed and 'title' in data.feed:
sourcename = data.feed['title']
sourcename = unescape_html(sourcename)
ids = []
news = []
links = []
for i in data.entries:
date = i.get('published_parsed', i.get('updated_parsed', ''))
if date:
date = datetime.fromtimestamp(time.mktime(date))
if datetime.today() - date > timedelta(
hours=config.BACK_HOURS + 6):
break
link, self.fact.cache_urls = yield clean_redir_urls(
i.get('link', ''), self.fact.cache_urls)
if not link.startswith('http'):
link = "%s/%s" % (url[:url.find('/', 8)], link.lstrip('/'))
if link in links:
continue
links.append(link)
title = i.get('title', '').replace('\n', ' ')
try:
title = unescape_html(title)
except:
pass
_id = md5(
("%s:%s:%s" % (self.fact.channel, link,
title.lower())).encode('utf-8')).hexdigest()
ids.append(_id)
news.append({
'_id': _id,
'channel': self.fact.channel,
'message': title,
'link': link,
'date': date,
'timestamp': datetime.today(),
'source': url,
'sourcename': sourcename
})
existings = yield self.fact.db['news'].find(
{
'channel': self.fact.channel,
'_id': {
'$in': ids
}
},
fields=['_id'],
filter=sortdesc('_id'))
existing = [n['_id'] for n in existings]
new = [n for n in news if n['_id'] not in existing]
if new:
new.reverse()
new = new[:5]
try:
yield self.fact.db['news'].insert(new, safe=True)
except Exception as e:
self._handle_error(e, "recording news batch", url)
self.fact.ircclient._send_message(
[(True, "[%s] %s" %
(n['sourcename'].encode('utf-8'), self.format_tweet(n)))
for n in new], self.fact.channel)
returnD(True)
def process_dms(self, listdms, user):
if not listdms:
returnD(False)
ids = []
dms = []
try:
listdms = listdms["events"]
assert(isinstance(listdms, list))
except:
self.log("downloading DMs: %s" % listdms, error=True)
returnD(False)
for i in listdms:
try:
date = parse_timestamp(i.get('created_timestamp', ''))
if datetime.today() - date > timedelta(hours=config.BACK_HOURS):
break
except Exception as e:
self.log("processing DM %s: %s %s" % (i.get('created_timestamp'), type(e), e), error=True)
continue
tid = long(i.get('id', ''))
msg = i.get('message_create', {})
if tid and msg:
ids.append(tid)
sender = msg.get('sender_id', '')
target = msg.get('target', {}).get('recipient_id', '')
dm, self.fact.cache_urls = yield clean_redir_urls(msg.get('message_data', {}).get('text', '').replace('\n', ' '), self.fact.cache_urls)
dms.append({'_id': "%s:%s" % (self.fact.channel, tid), 'channel': self.fact.channel, 'id': tid, 'user': user, 'sender_id': sender, 'target_id': target, 'message': dm, 'date': date, 'timestamp': datetime.today()})
existings = yield self.fact.db['dms'].find({'channel': self.fact.channel, 'id': {'$in': ids}}, fields=['_id'], filter=sortdesc('id'))
existing = [t['_id'] for t in existings]
news = [t for t in dms if t['_id'] not in existing]
if news:
news.reverse()
conf = chanconf(self.fact.channel)
conn = Microblog('twitter', conf, bearer_token=conf["oauth2"])
res = yield conn.resolve_userids([n["sender_id"] for n in news] + [n["target_id"] for n in news])
if "ERROR 429" in res or "ERROR 404" in res or not isinstance(res, list):
self.log("resolving users from DMs %s: %s %s" % (res, type(e), e), error=True)
returnD(False)
users = dict((u['id_str'], u['screen_name']) for u in res)
for n in news:
n["screenname"] = users.get(n["sender_id"], "unknown")
n["sender"] = n["screenname"].lower()
n["target_screenname"] = users.get(n["target_id"], "unknown")
n["target"] = n["target_screenname"].lower()
yield self.fact.db['dms'].insert(news, safe=True)
self.fact.ircclient._send_message([(True, "[DM] @%s ➜ @%s: %s — https://twitter.com/%s" % (n['screenname'].encode('utf-8'), n['target_screenname'].encode('utf-8'), n['message'].encode('utf-8'), n['screenname'].encode('utf-8'))) for n in news], self.fact.channel)
returnD(True)
