def _send_via_protocol(self, siteprotocol, command, channel, nick, **kwargs): conf = chanconf(channel) if not chan_has_protocol(channel, siteprotocol, conf): return "No %s account is set for this channel." % siteprotocol if 'text' in kwargs: kwargs['text'], nolimit = self._match_reg(kwargs['text'], self.re_nolimit) kwargs['text'], force = self._match_reg(kwargs['text'], self.re_force) try: ct = countchars(kwargs['text']) except: ct = 100 if ct < 30 and not nolimit: return "Do you really want to send such a short message? (%s chars) add --nolimit to override" % ct elif ct > 140 and siteprotocol == "twitter" and not nolimit: return "[%s] Sorry, but that's too long (%s characters) add --nolimit to override" % (site.protocol, ct) if command in ['microblog', 'retweet']: kwargs['channel'] = channel if "TWITTER" in conf: conn = Microblog("twitter", conf) if 'text' in kwargs and not force and command != "directmsg": bl, self.twitter_users, msg = conn.test_microblog_users(kwargs['text'], self.twitter_users) if not bl: return "[%s] %s" % (siteprotocol, msg) if site.protocol == "identica": conn = Microblog(siteprotocol, conf) command = getattr(conn, command, None) return command(**kwargs)
def follow_stream(self, conf, follow, track): conn = Microblog("twitter", conf, streaming=True) ct = 0 tweets = [] flush = time.time() + 29 try: for tweet in conn.search_stream(follow, track): if self.fact.status.startswith("clos"): self._flush_tweets(tweets) self.log("Feeder closed.", "stream", hint=True) break elif not tweet or not tweet.get('text'): if tweet and not tweet.get('delete'): self.log(tweet, "stream") continue elif tweet.get("disconnect"): self._flush_tweets(tweets) self.log("Disconnected %s" % tweet, "stream", error=True) break tweets.append(tweet) ct += 1 if ct > 9 or time.time() > flush: self._flush_tweets(tweets) ct = 0 tweets = [] flush = time.time() + 29 except Exception as e: self.log(e, "stream", error=True) self._handle_error(e.traceback, "while followoing", "stream") return
def start_stream(self, conf): if not self.fact.__init_timeout__(): returnD(False) queries = yield self.fact.db['feeds'].find({'database': 'tweets', 'channel': self.fact.channel}, fields=['query']) track = [] skip = [] k = 0 for query in queries: q = str(query['query'].encode('utf-8')).lower() # queries starting with @ should return only tweets from corresponding user, stream doesn not know how to handle this so skip if self.re_twitter_account.match(q): continue elif " OR " in q or " -" in q or '"' in q or len(q) > 60 or len(q) < 6: skip.append(q) continue track.append(q) k += 1 if k > 395: break if self.fact.twuser not in track: track.append(self.fact.twuser) if len(skip): self.log("Skipping unprocessable queries for streaming: « %s »" % " » | « ".join(skip), hint=True) self.log("Start search streaming for: « %s »" % " » | « ".join(track), hint=True) conn = Microblog("twitter", conf, bearer_token=self.fact.twitter_token) # tries to find users corresponding with queries to follow with stream users, self.fact.ircclient.twitter['users'] = conn.lookup_users(track, self.fact.ircclient.twitter['users']) deferToThreadPool(reactor, self.threadpool, self.follow_stream, conf, users.values(), track) self.depiler = LoopingCall(self.flush_tweets) self.depiler.start(1) returnD(True)
def start_stream(self, conf): self.db.authenticate(config.MONGODB['USER'], config.MONGODB['PSWD']) queries = list(self.db["feeds"].find({'database': "tweets", 'channel': self.fact.channel}, fields=['query'])) track = [] follow = [] skip = [] k = 0 f = 0 for query in queries: q = str(query['query'].encode('utf-8')) if self.re_twitter_account.match(q): q = q.lstrip('@') follow.append(q) f += 1 elif " OR " in q or " -" in q or '"' in q or len(q) > 60: skip.append(q) continue track.append(q) k += 1 if k > 395 or f > 4995: break user = conf["TWITTER"]["USER"] if user not in follow: follow.append(user) if user not in track: track.append(user) if len(skip): self.log("Skipping unprocessable queries for streaming: « %s »" % " » | « ".join(skip), "stream", hint=True) self.log("Start search streaming for: « %s »" % " » | « ".join(track), "stream", hint=True) conn = Microblog("twitter", conf, bearer_token=self.fact.twitter_token) users, self.fact.ircclient.twitter_users = conn.search_users(follow, self.fact.ircclient.twitter_users) return deferToThreadPool(reactor, self.threadpool, self.follow_stream, conf, users.values(), track)
def follow_stream(self, conf, follow, track): conn = Microblog("twitter", conf, streaming=True) try: for tweet in conn.search_stream(follow, track): self.fact.update_timeout() if self.fact.status == "closed": break if tweet: if tweet.get("disconnect") or tweet.get("hangup"): self.log("Disconnected %s" % ("(timeout)" if tweet.get("heartbeat_timeout") else tweet), error=True) break if tweet.get('timeout'): continue # heartbeat if tweet.get('text'): self.pile.insert(0, tweet) else: try: self.fact.db['tweets'].update(spec={'id': tweet['delete']['status']['id']}, document={'$set': {'deleted': True}}, multi=True) if config.DEBUG: self.log("Mark a tweet as deleted: %s" % tweet['delete']['status']['id'], hint=True) except: if config.DEBUG: self.log(tweet, hint=True) except socket.error as e: self.log("Stream lost connection with %s: %s" % (type(e), e), error=True) except Exception as e: if str(e).strip(): self.log("Stream crashed with %s: %s" % (type(e), e), error=True) else: self._handle_error(failure.Failure(e), "following", "stream") self.depiler.stop() self.flush_tweets() self.log("Feeder closed.", hint=True) if self.fact.status != "closed": self.fact.status = "stopped"
def _rt_on_identica(self, tweet_id, conf, channel, nick): conn = Microblog('twitter', conf) res = conn.show_status(tweet_id) if res and 'user' in res and 'screen_name' in res['user'] and 'text' in res: tweet = "♻ @%s: %s" % (res['user']['screen_name'].encode('utf-8'), res['text'].encode('utf-8')) if countchars(tweet) > 140: tweet = "%s…" % tweet[:139] return self._send_via_protocol('identica', 'microblog', channel, nick, text=tweet) return "[identica] Cannot find tweet %s on Twitter." % tweet_id
def search_twitter(self, data, query, max_id=None, page=0, randorder=None): if page and randorder: try: query = getFeeds(self.fact.channel, "tweets", self.db, randorder=randorder)[page] except: return None if config.DEBUG: text = unquote(query) if max_id: text += " before id %s" % max_id self.log("Query Twitter search for %s" % text, "search") conn = Microblog('twitter', chanconf(self.fact.channel), bearer_token=self.fact.twitter_token) return conn.search(query, max_id=max_id)
def search_twitter(self, data, query, max_id=None, page=0, randorder=None): if page and randorder: try: query = yield getFeeds(self.fact.db, self.fact.channel, "tweets", randorder=randorder) query = query[page] except Exception as e: returnD(False) if config.DEBUG: text = unquote(query) if max_id: text = "%s before id %s" % (text, max_id.encode('utf-8')) self.log("Query Twitter search for %s" % text) conn = Microblog('twitter', chanconf(self.fact.channel), bearer_token=self.fact.twitter_token) res = conn.search(query, max_id=max_id) returnD(res)
def follow_stream(self, conf, follow, track): conn = Microblog("twitter", conf, streaming=True) try: for tweet in conn.search_stream(follow, track): self.fact.update_timeout() if self.fact.status == "closed": break if tweet: if tweet.get("disconnect") or tweet.get("hangup"): self.log("Disconnected %s" % ("(timeout)" if tweet.get("heartbeat_timeout") else tweet), error=True) break if tweet.get('timeout'): continue # heartbeat if tweet.get('id_str'): tweet = reformat_extended_tweets(tweet) self.pile.insert(0, tweet) else: try: self.fact.db['tweets'].update( spec={'id': tweet['delete']['status']['id']}, document={'$set': { 'deleted': True }}, multi=True) if config.DEBUG: self.log("Mark a tweet as deleted: %s" % tweet['delete']['status']['id'], hint=True) except: if config.DEBUG: self.log(tweet, hint=True) except socket.error as e: self.log("Stream lost connection with %s: %s" % (type(e), e), error=True) except Exception as e: if str(e).strip(): self.log("Stream crashed with %s: %s" % (type(e), e), error=True) else: self._handle_error(failure.Failure(e), "following", "stream") self.depiler.stop() self.flush_tweets() self.log("Feeder closed.", hint=True) if self.fact.status != "closed": self.fact.status = "stopped"
def process_dms(self, listdms, user): if not listdms: returnD(False) ids = [] dms = [] try: listdms = listdms["events"] assert(isinstance(listdms, list)) except: self.log("downloading DMs: %s" % listdms, error=True) returnD(False) for i in listdms: try: date = parse_timestamp(i.get('created_timestamp', '')) if datetime.today() - date > timedelta(hours=config.BACK_HOURS): break except Exception as e: self.log("processing DM %s: %s %s" % (i.get('created_timestamp'), type(e), e), error=True) continue tid = long(i.get('id', '')) msg = i.get('message_create', {}) if tid and msg: ids.append(tid) sender = msg.get('sender_id', '') target = msg.get('target', {}).get('recipient_id', '') dm, self.fact.cache_urls = yield clean_redir_urls(msg.get('message_data', {}).get('text', '').replace('\n', ' '), self.fact.cache_urls) dms.append({'_id': "%s:%s" % (self.fact.channel, tid), 'channel': self.fact.channel, 'id': tid, 'user': user, 'sender_id': sender, 'target_id': target, 'message': dm, 'date': date, 'timestamp': datetime.today()}) existings = yield self.fact.db['dms'].find({'channel': self.fact.channel, 'id': {'$in': ids}}, fields=['_id'], filter=sortdesc('id')) existing = [t['_id'] for t in existings] news = [t for t in dms if t['_id'] not in existing] if news: news.reverse() conf = chanconf(self.fact.channel) conn = Microblog('twitter', conf, bearer_token=conf["oauth2"]) res = yield conn.resolve_userids([n["sender_id"] for n in news] + [n["target_id"] for n in news]) if "ERROR 429" in res or "ERROR 404" in res or not isinstance(res, list): self.log("resolving users from DMs %s: %s %s" % (res, type(e), e), error=True) returnD(False) users = dict((u['id_str'], u['screen_name']) for u in res) for n in news: n["screenname"] = users.get(n["sender_id"], "unknown") n["sender"] = n["screenname"].lower() n["target_screenname"] = users.get(n["target_id"], "unknown") n["target"] = n["target_screenname"].lower() yield self.fact.db['dms'].insert(news, safe=True) self.fact.ircclient._send_message([(True, "[DM] @%s ➜ @%s: %s — https://twitter.com/%s" % (n['screenname'].encode('utf-8'), n['target_screenname'].encode('utf-8'), n['message'].encode('utf-8'), n['screenname'].encode('utf-8'))) for n in news], self.fact.channel) returnD(True)
def process_stats(self, stats, user): # Update followers list conf = chanconf(self.fact.channel) conn = Microblog('twitter', conf, bearer_token=conf["oauth2"]) lost = yield conn.update_followers(self.fact.db) ct = len(lost) if ct: self.fact.ircclient._send_message('[twitter] Lost %s follower%s: %s%s' % (ct, "s" if ct>1 else "", format_4_followers(lost), "…" if ct>4 else ""), self.fact.channel) # Update stats if not stats: returnD(False) stats, last, timestamp = stats if not stats or type(stats) is str: returnD(False) if not last: last = {'tweets': 0, 'followers': 0} since = timestamp - timedelta(hours=1) else: since = last['timestamp'] if 'lists' not in last: last['lists'] = 0 re_match_rts = re.compile(u'(([MLR]T|%s|♺)\s*)+@?%s' % (QUOTE_CHARS, user), re.I) rts = yield self.fact.db['tweets'].find({'channel': self.fact.channel, 'message': re_match_rts, 'timestamp': {'$gte': since}}, fields=['_id']) nb_rts = len(rts) nb_fols = yield count_followers(user) stat = {'user': user, 'timestamp': timestamp, 'tweets': stats.get('statuses_count', last['tweets']), 'followers': nb_fols, 'rts_last_hour': nb_rts, 'lists': stats.get('listed_count', last['lists'])} yield self.fact.db['stats'].insert(stat) weekday = timestamp.weekday() laststats = Stats(user) if chan_displays_stats(self.fact.channel) and ((timestamp.hour == 13 and weekday < 5) or timestamp.hour == 18): stats = yield laststats.print_last() self.fact.ircclient._send_message(stats, self.fact.channel) last_tweet = yield self.fact.db['tweets'].find({'channel': self.fact.channel, 'user': user}, fields=['date'], limit=1, filter=sortdesc('timestamp')) if chan_displays_stats(self.fact.channel) and last_tweet and timestamp - last_tweet[0]['date'] > timedelta(days=3) and (timestamp.hour == 11 or timestamp.hour == 17) and weekday < 5: reactor.callFromThread(reactor.callLater, 3, self.fact.ircclient._send_message, "[FYI] No tweet was sent since %s days." % (timestamp - last_tweet[0]['date']).days, self.fact.channel) reactor.callFromThread(reactor.callLater, 1, laststats.dump_data) returnD(True)
def start_twitter(self, name, conf, user): if not self.fact.__init_timeout__(): returnD(False) d = succeed(Microblog('twitter', conf, bearer_token=self.fact.twitter_token)) if config.DEBUG: self.log("Query @%s's %s" % (user, name)) def passs(*args, **kwargs): raise Exception("No process existing for %s" % name) source = getattr(Microblog, 'get_%s' % name, passs) processor = getattr(self, 'process_%s' % name, passs) d.addCallback(source, retweets_processed=self.fact.retweets_processed, bearer_token=self.fact.twitter_token) d.addErrback(self._handle_error, "downloading %s for" % name, user) d.addCallback(check_twitter_results) d.addErrback(self._handle_error, "examining %s for" % name, user) d.addCallback(processor, user.lower()) d.addErrback(self._handle_error, "working on %s for" % name, user) d.addCallback(self.end_twitter) return d
try: from gazouilleur.identica_auth_config import identica_auth [ identica_auth[conf['IDENTICA']['USER'].lower()] for conf in config.CHANNELS.values() if "IDENTICA" in conf ] except (ImportError, KeyError) as e: logerr( "Could not find `gazouilleur/identica_auth_config.py` with configuration for %s.\nERROR: Please run `python bin/auth_identica.py` to generate your OAuth Identi.ca keys and create it automatically.\n" % e) exit(1) from gazouilleur.lib.microblog import Microblog for chan, conf in config.CHANNELS.iteritems(): if "IDENTICA" not in conf: continue conn = Microblog("identica", conf) try: from urllib2 import urlopen urlopen("https://identi.ca", timeout=15) if not conn.ping(): logerr( "Cannot connect to Identi.ca with the auth configuration provided in `gazouilleur/identica_auth_config.py` for channel %s and user @%s.\nERROR: Please rerun `python bin/auth_identica.py` to generate your OAuth Identi.ca keys.\n" % (chan, conf["IDENTICA"]["USER"].lower())) exit(1) except: stderr.write( colorize( "WARNING: Identi.ca seems down, bypassing related tests.\n", 'red', style='bold'))
logerr("Cannot connect to database %s in MongoDB.\nERROR: Please check the database and its users are created,\nERROR: or run `bash bin/configureDB.sh` to create or update them automatically (or configureDB-mongo3.sh when using MongoDB v3+).\n%s\n" % (config.MONGODB['DATABASE'], e)) exit(1) # Check Identi.ca config if [1 for c in config.CHANNELS.values() if "IDENTICA" in c]: try: from gazouilleur.identica_auth_config import identica_auth [identica_auth[conf['IDENTICA']['USER'].lower()] for conf in config.CHANNELS.values() if "IDENTICA" in conf] except (ImportError, KeyError) as e: logerr("Could not find `gazouilleur/identica_auth_config.py` with configuration for %s.\nERROR: Please run `python bin/auth_identica.py` to generate your OAuth Identi.ca keys and create it automatically.\n" % e) exit(1) from gazouilleur.lib.microblog import Microblog for chan, conf in config.CHANNELS.iteritems(): if "IDENTICA" not in conf: continue conn = Microblog("identica", conf) try: from urllib2 import urlopen urlopen("https://identi.ca", timeout=15) if not conn.ping(): logerr("Cannot connect to Identi.ca with the auth configuration provided in `gazouilleur/identica_auth_config.py` for channel %s and user @%s.\nERROR: Please rerun `python bin/auth_identica.py` to generate your OAuth Identi.ca keys.\n" % (chan, conf["IDENTICA"]["USER"].lower())) exit(1) except: stderr.write(colorize("WARNING: Identi.ca seems down, bypassing related tests.\n", 'red', style='bold')) # Check Twitter config for chan, conf in config.CHANNELS.iteritems(): if "TWITTER" not in conf: continue conn = Microblog("twitter", conf) if not conn.ping():