def _send_via_protocol(self, siteprotocol, command, channel, nick, **kwargs):
conf = chanconf(channel)
if not chan_has_protocol(channel, siteprotocol, conf):
return "No %s account is set for this channel." % siteprotocol
if 'text' in kwargs:
kwargs['text'], nolimit = self._match_reg(kwargs['text'], self.re_nolimit)
kwargs['text'], force = self._match_reg(kwargs['text'], self.re_force)
try:
ct = countchars(kwargs['text'])
except:
ct = 100
if ct < 30 and not nolimit:
return "Do you really want to send such a short message? (%s chars) add --nolimit to override" % ct
elif ct > 140 and siteprotocol == "twitter" and not nolimit:
return "[%s] Sorry, but that's too long (%s characters) add --nolimit to override" % (site.protocol, ct)
if command in ['microblog', 'retweet']:
kwargs['channel'] = channel
if "TWITTER" in conf:
conn = Microblog("twitter", conf)
if 'text' in kwargs and not force and command != "directmsg":
bl, self.twitter_users, msg = conn.test_microblog_users(kwargs['text'], self.twitter_users)
if not bl:
return "[%s] %s" % (siteprotocol, msg)
if site.protocol == "identica":
conn = Microblog(siteprotocol, conf)
command = getattr(conn, command, None)
return command(**kwargs)
def follow_stream(self, conf, follow, track):
conn = Microblog("twitter", conf, streaming=True)
ct = 0
tweets = []
flush = time.time() + 29
try:
for tweet in conn.search_stream(follow, track):
if self.fact.status.startswith("clos"):
self._flush_tweets(tweets)
self.log("Feeder closed.", "stream", hint=True)
break
elif not tweet or not tweet.get('text'):
if tweet and not tweet.get('delete'):
self.log(tweet, "stream")
continue
elif tweet.get("disconnect"):
self._flush_tweets(tweets)
self.log("Disconnected %s" % tweet, "stream", error=True)
break
tweets.append(tweet)
ct += 1
if ct > 9 or time.time() > flush:
self._flush_tweets(tweets)
ct = 0
tweets = []
flush = time.time() + 29
except Exception as e:
self.log(e, "stream", error=True)
self._handle_error(e.traceback, "while followoing", "stream")
return
def start_stream(self, conf):
if not self.fact.__init_timeout__():
returnD(False)
queries = yield self.fact.db['feeds'].find({'database': 'tweets', 'channel': self.fact.channel}, fields=['query'])
track = []
skip = []
k = 0
for query in queries:
q = str(query['query'].encode('utf-8')).lower()
# queries starting with @ should return only tweets from corresponding user, stream doesn not know how to handle this so skip
if self.re_twitter_account.match(q):
continue
elif " OR " in q or " -" in q or '"' in q or len(q) > 60 or len(q) < 6:
skip.append(q)
continue
track.append(q)
k += 1
if k > 395:
break
if self.fact.twuser not in track:
track.append(self.fact.twuser)
if len(skip):
self.log("Skipping unprocessable queries for streaming: « %s »" % " » | « ".join(skip), hint=True)
self.log("Start search streaming for: « %s »" % " » | « ".join(track), hint=True)
conn = Microblog("twitter", conf, bearer_token=self.fact.twitter_token)
# tries to find users corresponding with queries to follow with stream
users, self.fact.ircclient.twitter['users'] = conn.lookup_users(track, self.fact.ircclient.twitter['users'])
deferToThreadPool(reactor, self.threadpool, self.follow_stream, conf, users.values(), track)
self.depiler = LoopingCall(self.flush_tweets)
self.depiler.start(1)
returnD(True)
def start_stream(self, conf):
self.db.authenticate(config.MONGODB['USER'], config.MONGODB['PSWD'])
queries = list(self.db["feeds"].find({'database': "tweets", 'channel': self.fact.channel}, fields=['query']))
track = []
follow = []
skip = []
k = 0
f = 0
for query in queries:
q = str(query['query'].encode('utf-8'))
if self.re_twitter_account.match(q):
q = q.lstrip('@')
follow.append(q)
f += 1
elif " OR " in q or " -" in q or '"' in q or len(q) > 60:
skip.append(q)
continue
track.append(q)
k += 1
if k > 395 or f > 4995:
break
user = conf["TWITTER"]["USER"]
if user not in follow:
follow.append(user)
if user not in track:
track.append(user)
if len(skip):
self.log("Skipping unprocessable queries for streaming: « %s »" % " » | « ".join(skip), "stream", hint=True)
self.log("Start search streaming for: « %s »" % " » | « ".join(track), "stream", hint=True)
conn = Microblog("twitter", conf, bearer_token=self.fact.twitter_token)
users, self.fact.ircclient.twitter_users = conn.search_users(follow, self.fact.ircclient.twitter_users)
return deferToThreadPool(reactor, self.threadpool, self.follow_stream, conf, users.values(), track)
def start_stream(self, conf):
if not self.fact.__init_timeout__():
returnD(False)
queries = yield self.fact.db['feeds'].find({'database': 'tweets', 'channel': self.fact.channel}, fields=['query'])
track = []
skip = []
k = 0
for query in queries:
q = str(query['query'].encode('utf-8')).lower()
# queries starting with @ should return only tweets from corresponding user, stream doesn not know how to handle this so skip
if self.re_twitter_account.match(q):
continue
elif " OR " in q or " -" in q or '"' in q or len(q) > 60 or len(q) < 6:
skip.append(q)
continue
track.append(q)
k += 1
if k > 395:
break
if self.fact.twuser not in track:
track.append(self.fact.twuser)
if len(skip):
self.log("Skipping unprocessable queries for streaming: « %s »" % " » | « ".join(skip), hint=True)
self.log("Start search streaming for: « %s »" % " » | « ".join(track), hint=True)
conn = Microblog("twitter", conf, bearer_token=self.fact.twitter_token)
# tries to find users corresponding with queries to follow with stream
users, self.fact.ircclient.twitter['users'] = conn.lookup_users(track, self.fact.ircclient.twitter['users'])
deferToThreadPool(reactor, self.threadpool, self.follow_stream, conf, users.values(), track)
self.depiler = LoopingCall(self.flush_tweets)
self.depiler.start(1)
returnD(True)
def follow_stream(self, conf, follow, track):
conn = Microblog("twitter", conf, streaming=True)
try:
for tweet in conn.search_stream(follow, track):
self.fact.update_timeout()
if self.fact.status == "closed":
break
if tweet:
if tweet.get("disconnect") or tweet.get("hangup"):
self.log("Disconnected %s" % ("(timeout)" if tweet.get("heartbeat_timeout") else tweet), error=True)
break
if tweet.get('timeout'):
continue # heartbeat
if tweet.get('text'):
self.pile.insert(0, tweet)
else:
try:
self.fact.db['tweets'].update(spec={'id': tweet['delete']['status']['id']}, document={'$set': {'deleted': True}}, multi=True)
if config.DEBUG:
self.log("Mark a tweet as deleted: %s" % tweet['delete']['status']['id'], hint=True)
except:
if config.DEBUG:
self.log(tweet, hint=True)
except socket.error as e:
self.log("Stream lost connection with %s: %s" % (type(e), e), error=True)
except Exception as e:
if str(e).strip():
self.log("Stream crashed with %s: %s" % (type(e), e), error=True)
else:
self._handle_error(failure.Failure(e), "following", "stream")
self.depiler.stop()
self.flush_tweets()
self.log("Feeder closed.", hint=True)
if self.fact.status != "closed":
self.fact.status = "stopped"
def _rt_on_identica(self, tweet_id, conf, channel, nick):
conn = Microblog('twitter', conf)
res = conn.show_status(tweet_id)
if res and 'user' in res and 'screen_name' in res['user'] and 'text' in res:
tweet = "♻ @%s: %s" % (res['user']['screen_name'].encode('utf-8'), res['text'].encode('utf-8'))
if countchars(tweet) > 140:
tweet = "%s…" % tweet[:139]
return self._send_via_protocol('identica', 'microblog', channel, nick, text=tweet)
return "[identica] Cannot find tweet %s on Twitter." % tweet_id
def search_twitter(self, data, query, max_id=None, page=0, randorder=None):
if page and randorder:
try:
query = getFeeds(self.fact.channel, "tweets", self.db, randorder=randorder)[page]
except:
return None
if config.DEBUG:
text = unquote(query)
if max_id:
text += " before id %s" % max_id
self.log("Query Twitter search for %s" % text, "search")
conn = Microblog('twitter', chanconf(self.fact.channel), bearer_token=self.fact.twitter_token)
return conn.search(query, max_id=max_id)
def search_twitter(self, data, query, max_id=None, page=0, randorder=None):
if page and randorder:
try:
query = yield getFeeds(self.fact.db, self.fact.channel, "tweets", randorder=randorder)
query = query[page]
except Exception as e:
returnD(False)
if config.DEBUG:
text = unquote(query)
if max_id:
text = "%s before id %s" % (text, max_id.encode('utf-8'))
self.log("Query Twitter search for %s" % text)
conn = Microblog('twitter', chanconf(self.fact.channel), bearer_token=self.fact.twitter_token)
res = conn.search(query, max_id=max_id)
returnD(res)
def search_twitter(self, data, query, max_id=None, page=0, randorder=None):
if page and randorder:
try:
query = yield getFeeds(self.fact.db, self.fact.channel, "tweets", randorder=randorder)
query = query[page]
except Exception as e:
returnD(False)
if config.DEBUG:
text = unquote(query)
if max_id:
text = "%s before id %s" % (text, max_id.encode('utf-8'))
self.log("Query Twitter search for %s" % text)
conn = Microblog('twitter', chanconf(self.fact.channel), bearer_token=self.fact.twitter_token)
res = conn.search(query, max_id=max_id)
returnD(res)
def follow_stream(self, conf, follow, track):
conn = Microblog("twitter", conf, streaming=True)
try:
for tweet in conn.search_stream(follow, track):
self.fact.update_timeout()
if self.fact.status == "closed":
break
if tweet:
if tweet.get("disconnect") or tweet.get("hangup"):
self.log("Disconnected %s" %
("(timeout)" if tweet.get("heartbeat_timeout")
else tweet),
error=True)
break
if tweet.get('timeout'):
continue # heartbeat
if tweet.get('id_str'):
tweet = reformat_extended_tweets(tweet)
self.pile.insert(0, tweet)
else:
try:
self.fact.db['tweets'].update(
spec={'id': tweet['delete']['status']['id']},
document={'$set': {
'deleted': True
}},
multi=True)
if config.DEBUG:
self.log("Mark a tweet as deleted: %s" %
tweet['delete']['status']['id'],
hint=True)
except:
if config.DEBUG:
self.log(tweet, hint=True)
except socket.error as e:
self.log("Stream lost connection with %s: %s" % (type(e), e),
error=True)
except Exception as e:
if str(e).strip():
self.log("Stream crashed with %s: %s" % (type(e), e),
error=True)
else:
self._handle_error(failure.Failure(e), "following", "stream")
self.depiler.stop()
self.flush_tweets()
self.log("Feeder closed.", hint=True)
if self.fact.status != "closed":
self.fact.status = "stopped"
def process_dms(self, listdms, user):
if not listdms:
returnD(False)
ids = []
dms = []
try:
listdms = listdms["events"]
assert(isinstance(listdms, list))
except:
self.log("downloading DMs: %s" % listdms, error=True)
returnD(False)
for i in listdms:
try:
date = parse_timestamp(i.get('created_timestamp', ''))
if datetime.today() - date > timedelta(hours=config.BACK_HOURS):
break
except Exception as e:
self.log("processing DM %s: %s %s" % (i.get('created_timestamp'), type(e), e), error=True)
continue
tid = long(i.get('id', ''))
msg = i.get('message_create', {})
if tid and msg:
ids.append(tid)
sender = msg.get('sender_id', '')
target = msg.get('target', {}).get('recipient_id', '')
dm, self.fact.cache_urls = yield clean_redir_urls(msg.get('message_data', {}).get('text', '').replace('\n', ' '), self.fact.cache_urls)
dms.append({'_id': "%s:%s" % (self.fact.channel, tid), 'channel': self.fact.channel, 'id': tid, 'user': user, 'sender_id': sender, 'target_id': target, 'message': dm, 'date': date, 'timestamp': datetime.today()})
existings = yield self.fact.db['dms'].find({'channel': self.fact.channel, 'id': {'$in': ids}}, fields=['_id'], filter=sortdesc('id'))
existing = [t['_id'] for t in existings]
news = [t for t in dms if t['_id'] not in existing]
if news:
news.reverse()
conf = chanconf(self.fact.channel)
conn = Microblog('twitter', conf, bearer_token=conf["oauth2"])
res = yield conn.resolve_userids([n["sender_id"] for n in news] + [n["target_id"] for n in news])
if "ERROR 429" in res or "ERROR 404" in res or not isinstance(res, list):
self.log("resolving users from DMs %s: %s %s" % (res, type(e), e), error=True)
returnD(False)
users = dict((u['id_str'], u['screen_name']) for u in res)
for n in news:
n["screenname"] = users.get(n["sender_id"], "unknown")
n["sender"] = n["screenname"].lower()
n["target_screenname"] = users.get(n["target_id"], "unknown")
n["target"] = n["target_screenname"].lower()
yield self.fact.db['dms'].insert(news, safe=True)
self.fact.ircclient._send_message([(True, "[DM] @%s ➜ @%s: %s — https://twitter.com/%s" % (n['screenname'].encode('utf-8'), n['target_screenname'].encode('utf-8'), n['message'].encode('utf-8'), n['screenname'].encode('utf-8'))) for n in news], self.fact.channel)
returnD(True)
def process_stats(self, stats, user):
# Update followers list
conf = chanconf(self.fact.channel)
conn = Microblog('twitter', conf, bearer_token=conf["oauth2"])
lost = yield conn.update_followers(self.fact.db)
ct = len(lost)
if ct:
self.fact.ircclient._send_message('[twitter] Lost %s follower%s: %s%s' % (ct, "s" if ct>1 else "", format_4_followers(lost), "…" if ct>4 else ""), self.fact.channel)
# Update stats
if not stats:
returnD(False)
stats, last, timestamp = stats
if not stats or type(stats) is str:
returnD(False)
if not last:
last = {'tweets': 0, 'followers': 0}
since = timestamp - timedelta(hours=1)
else:
since = last['timestamp']
if 'lists' not in last:
last['lists'] = 0
re_match_rts = re.compile(u'(([MLR]T|%s|♺)\s*)+@?%s' % (QUOTE_CHARS, user), re.I)
rts = yield self.fact.db['tweets'].find({'channel': self.fact.channel, 'message': re_match_rts, 'timestamp': {'$gte': since}}, fields=['_id'])
nb_rts = len(rts)
nb_fols = yield count_followers(user)
stat = {'user': user, 'timestamp': timestamp, 'tweets': stats.get('statuses_count', last['tweets']), 'followers': nb_fols, 'rts_last_hour': nb_rts, 'lists': stats.get('listed_count', last['lists'])}
yield self.fact.db['stats'].insert(stat)
weekday = timestamp.weekday()
laststats = Stats(user)
if chan_displays_stats(self.fact.channel) and ((timestamp.hour == 13 and weekday < 5) or timestamp.hour == 18):
stats = yield laststats.print_last()
self.fact.ircclient._send_message(stats, self.fact.channel)
last_tweet = yield self.fact.db['tweets'].find({'channel': self.fact.channel, 'user': user}, fields=['date'], limit=1, filter=sortdesc('timestamp'))
if chan_displays_stats(self.fact.channel) and last_tweet and timestamp - last_tweet[0]['date'] > timedelta(days=3) and (timestamp.hour == 11 or timestamp.hour == 17) and weekday < 5:
reactor.callFromThread(reactor.callLater, 3, self.fact.ircclient._send_message, "[FYI] No tweet was sent since %s days." % (timestamp - last_tweet[0]['date']).days, self.fact.channel)
reactor.callFromThread(reactor.callLater, 1, laststats.dump_data)
returnD(True)
def process_stats(self, stats, user):
# Update followers list
conf = chanconf(self.fact.channel)
conn = Microblog('twitter', conf, bearer_token=conf["oauth2"])
lost = yield conn.update_followers(self.fact.db)
ct = len(lost)
if ct:
self.fact.ircclient._send_message('[twitter] Lost %s follower%s: %s%s' % (ct, "s" if ct>1 else "", format_4_followers(lost), "…" if ct>4 else ""), self.fact.channel)
# Update stats
if not stats:
returnD(False)
stats, last, timestamp = stats
if not stats or type(stats) is str:
returnD(False)
if not last:
last = {'tweets': 0, 'followers': 0}
since = timestamp - timedelta(hours=1)
else:
since = last['timestamp']
if 'lists' not in last:
last['lists'] = 0
re_match_rts = re.compile(u'(([MLR]T|%s|♺)\s*)+@?%s' % (QUOTE_CHARS, user), re.I)
rts = yield self.fact.db['tweets'].find({'channel': self.fact.channel, 'message': re_match_rts, 'timestamp': {'$gte': since}}, fields=['_id'])
nb_rts = len(rts)
nb_fols = yield count_followers(user)
stat = {'user': user, 'timestamp': timestamp, 'tweets': stats.get('statuses_count', last['tweets']), 'followers': nb_fols, 'rts_last_hour': nb_rts, 'lists': stats.get('listed_count', last['lists'])}
yield self.fact.db['stats'].insert(stat)
weekday = timestamp.weekday()
laststats = Stats(user)
if chan_displays_stats(self.fact.channel) and ((timestamp.hour == 13 and weekday < 5) or timestamp.hour == 18):
stats = yield laststats.print_last()
self.fact.ircclient._send_message(stats, self.fact.channel)
last_tweet = yield self.fact.db['tweets'].find({'channel': self.fact.channel, 'user': user}, fields=['date'], limit=1, filter=sortdesc('timestamp'))
if chan_displays_stats(self.fact.channel) and last_tweet and timestamp - last_tweet[0]['date'] > timedelta(days=3) and (timestamp.hour == 11 or timestamp.hour == 17) and weekday < 5:
reactor.callFromThread(reactor.callLater, 3, self.fact.ircclient._send_message, "[FYI] No tweet was sent since %s days." % (timestamp - last_tweet[0]['date']).days, self.fact.channel)
reactor.callFromThread(reactor.callLater, 1, laststats.dump_data)
returnD(True)
def start_twitter(self, name, conf, user):
if not self.fact.__init_timeout__():
returnD(False)
d = succeed(Microblog('twitter', conf, bearer_token=self.fact.twitter_token))
if config.DEBUG:
self.log("Query @%s's %s" % (user, name))
def passs(*args, **kwargs):
raise Exception("No process existing for %s" % name)
source = getattr(Microblog, 'get_%s' % name, passs)
processor = getattr(self, 'process_%s' % name, passs)
d.addCallback(source, retweets_processed=self.fact.retweets_processed, bearer_token=self.fact.twitter_token)
d.addErrback(self._handle_error, "downloading %s for" % name, user)
d.addCallback(check_twitter_results)
d.addErrback(self._handle_error, "examining %s for" % name, user)
d.addCallback(processor, user.lower())
d.addErrback(self._handle_error, "working on %s for" % name, user)
d.addCallback(self.end_twitter)
return d
try:
from gazouilleur.identica_auth_config import identica_auth
[
identica_auth[conf['IDENTICA']['USER'].lower()]
for conf in config.CHANNELS.values() if "IDENTICA" in conf
]
except (ImportError, KeyError) as e:
logerr(
"Could not find `gazouilleur/identica_auth_config.py` with configuration for %s.\nERROR: Please run `python bin/auth_identica.py` to generate your OAuth Identi.ca keys and create it automatically.\n"
% e)
exit(1)
from gazouilleur.lib.microblog import Microblog
for chan, conf in config.CHANNELS.iteritems():
if "IDENTICA" not in conf:
continue
conn = Microblog("identica", conf)
try:
from urllib2 import urlopen
urlopen("https://identi.ca", timeout=15)
if not conn.ping():
logerr(
"Cannot connect to Identi.ca with the auth configuration provided in `gazouilleur/identica_auth_config.py` for channel %s and user @%s.\nERROR: Please rerun `python bin/auth_identica.py` to generate your OAuth Identi.ca keys.\n"
% (chan, conf["IDENTICA"]["USER"].lower()))
exit(1)
except:
stderr.write(
colorize(
"WARNING: Identi.ca seems down, bypassing related tests.\n",
'red',
style='bold'))
logerr("Cannot connect to database %s in MongoDB.\nERROR: Please check the database and its users are created,\nERROR: or run `bash bin/configureDB.sh` to create or update them automatically (or configureDB-mongo3.sh when using MongoDB v3+).\n%s\n" % (config.MONGODB['DATABASE'], e))
exit(1)
# Check Identi.ca config
if [1 for c in config.CHANNELS.values() if "IDENTICA" in c]:
try:
from gazouilleur.identica_auth_config import identica_auth
[identica_auth[conf['IDENTICA']['USER'].lower()] for conf in config.CHANNELS.values() if "IDENTICA" in conf]
except (ImportError, KeyError) as e:
logerr("Could not find `gazouilleur/identica_auth_config.py` with configuration for %s.\nERROR: Please run `python bin/auth_identica.py` to generate your OAuth Identi.ca keys and create it automatically.\n" % e)
exit(1)
from gazouilleur.lib.microblog import Microblog
for chan, conf in config.CHANNELS.iteritems():
if "IDENTICA" not in conf:
continue
conn = Microblog("identica", conf)
try:
from urllib2 import urlopen
urlopen("https://identi.ca", timeout=15)
if not conn.ping():
logerr("Cannot connect to Identi.ca with the auth configuration provided in `gazouilleur/identica_auth_config.py` for channel %s and user @%s.\nERROR: Please rerun `python bin/auth_identica.py` to generate your OAuth Identi.ca keys.\n" % (chan, conf["IDENTICA"]["USER"].lower()))
exit(1)
except:
stderr.write(colorize("WARNING: Identi.ca seems down, bypassing related tests.\n", 'red', style='bold'))
# Check Twitter config
for chan, conf in config.CHANNELS.iteritems():
if "TWITTER" not in conf:
continue
conn = Microblog("twitter", conf)
if not conn.ping():
