def update_account(account): pushed = 0 for tweet in twitterhandler.get_latest_tweets(account): tweet["deleted"] = False if databasehandler.write_tweet(account, tweet["id_str"], tweet, overwrite=False): pushed += 1 ok("Pushed latest tweets for " + str("account") + " into the database.", detail="Pushed " + str(pushed) + " new tweets.")
def load_flashes(): say("Loading latest flashes...") for accountPair in configuration.get_accounts(): say("Loading flashes from Twitter account " + accountPair[0] + " (" + accountPair[1] + ")") latest_tweets = twitter.get_latest_statuses(accountPair[0][1:]) for tweet in latest_tweets: url = None if "entities" in tweet: if "urls" in tweet['entities']: if len(tweet['entities']['urls']) > 0: url = tweet['entities']["urls"][-1]['expanded_url'] push_flash( generate_flash(tweet["text"], url, accountPair[1], tweet["id"], tweet["created_at"], accountPair[2]) ) say("Loaded " + str(len(latest_tweets)) + " flashes from " + accountPair[0]) ok("Loaded " + str(len(latest_flashes)) + " flashes")
def _push_notification_to_subscribers(subscriptions, data): userio.say("Sending notification to " + str(subscriptions.count()) + " subscribers in a new thread...") for subscription in subscriptions: try: webpush( subscription_info=subscription, data=data, vapid_private_key=configuration. get_vapid_public_private_key_pair()[1], vapid_claims=VAPID_CLAIMS, #gcm_key=configuration.get_gcm_key(), ) except Exception as e: userio.error(" ...unable to send notification: " + str(e)) db.subscriptions.remove(subscription) userio.say(" ...removed subscription!") userio.ok("Finished sending notification.") thread.exit()
def on_status(self, status): try: say("Detected flash...") if configuration.is_following(status.user.screen_name): say("...from a relevant source!") url = None if 'urls' in status.entities and len( status.entities['urls']) > 0: url = status.entities['urls'][0]["expanded_url"] flash = generate_flash( status.text, url, configuration.get_name(status.user.screen_name), status.id, status.created_at, configuration.get_channel(status.user.screen_name)) push_flash(flash) ok("Detected and pushed flash: " + str(flash)) else: say("...from an irrelevant source.") except Exception as e: error("Encountered an exception while processing a flash: " + str(e)) return True
def full_pass(account): if not databasehandler.is_account(account): error("Cannot scan account that doesn't exist!", detail="Offending account is " + str(account)) else: # pull all the tweet ids from the database archived_ids = databasehandler.get_tweets(account) # pull all the tweets from Twitter available_tweets = twitterhandler.get_all_tweets(account) available_ids = [tweet["id"] for tweet in available_tweets] deleted_candidates = [ idz for idz in archived_ids if int(idz) not in available_ids ] ok("Found " + str(len(deleted_candidates)) + " deletion candidates...") deleted = [] for candidate in deleted_candidates: if not twitterhandler.does_status_exist(candidate): ok("Tweet " + str(candidate) + " has been deleted!") deleted.append(candidate) ok("Found " + str(len(deleted)) + " deleted tweets!") # also write the new tweets to the database pushed = 0 for tweet in available_tweets: if databasehandler.write_tweet(account, tweet["id"], tweet, overwrite=False): pushed += 1 # and mark the deleted tweets as deleted for tweet_id in deleted: tweet = databasehandler.get_tweet(account, tweet_id) tweet["deleted"] = True databasehandler.write_tweet(account, tweet_id, tweet) ok("Also pushed " + str(pushed) + " new tweets to the database.") return deleted
def operate(): ok("Beginning operation...") for account in databasehandler.configuration["monitored_accounts"]: say("Writing the account data for " + str(account)) databasehandler.write_account(account, twitterhandler.get_account_data(account)) say("Writing the other data for " + str(account)) scanner.update_account(account) # this operation is done at the beginning of operation because # it sets up the database on the first run, and it also grabs # any tweets quickly, before the first full pass is run on each account iteration = 0 while iteration < sys.maxint: if iteration % databasehandler.configuration["intervals"][ "fullpass"] == 0: for account in databasehandler.configuration["monitored_accounts"]: ok("Running full pass for " + str(account)) deleted = scanner.full_pass(account) if len(deleted) > 0: deleted_tweets = [ databasehandler.get_tweet(account, tweet) for tweet in deleted ] notificationhandler.notify_deletions(account, deleted) if iteration % databasehandler.configuration["intervals"][ "account"] == 0: for account in databasehandler.configuration["monitored_accounts"]: ok("Running account update...") databasehandler.write_account( account, twitterhandler.get_account_data(account)) for account in databasehandler.configuration["monitored_accounts"]: ok("Running partial update pass for " + str(account)) deleted = scanner.update_account(account) iteration += 1 time.sleep(databasehandler.configuration["intervals"]["pass"]) operate() # in 292471208677 years...
"fullpass": 600, "account": 1200, "pass": 5 }, "notifications": { "email": "*****@*****.**" } } with open("configuration.json", "w") as configuration_file: json.dump(default_configuration, configuration_file, indent=4, sort_keys=True) ok("Configuration file created!", detail="The file is located at " + os.path.abspath("configuration.json")) configuration_file.close() ok("Please configure the file, then re-run citizen140.") sys.exit() # exit the system to allow user to configure if not os.path.exists("database"): # make the database, for first run os.mkdir("database") ok("Created database folder!") configuration = {} with open("configuration.json", "r") as configuration_file: configuration = json.load(configuration_file) configuration["monitored_accounts"] = [ str(account) for account in configuration["monitored_accounts"]
class ErrorHandler(tornado.web.RequestHandler): def write_error(self, status_code, **kwargs): try: stats.request(str(get_ip(self.request))) except: error("Errored while handling request IP -- still served...") self.render("pages/error.html", message=httplib.responses[status_code], error=status_code) def get(self): try: stats.request(str(get_ip(self.request))) except: error("Errored while handling request IP -- still served...") self.render("pages/error.html", message="Page not found", error="404") application = tornado.web.Application( [(r"/", IndexHandler), (r"/api", ApiHandler), (r"/stats", StatsHandler), (r'/static/(.*)$', tornado.web.StaticFileHandler, { 'path': "pages/static" })], default_handler_class=ErrorHandler) if __name__ == "__main__": flashes.go() ok("Starting webserver...") application.listen(8888) tornado.ioloop.IOLoop.instance().start()
def start_streamer(): ok("Starting streamer...") thread = Thread(target=streamer_entrypoint) thread.setDaemon(True) thread.start() ok("Streamer started!")