def post(self): say("Receiving new subscription: " + self.request.body) self.set_header("Content-Type", "application/json") if push.register_new_receiver( tornado.escape.json_decode(self.request.body)): self.write(json.dumps({"response": "OK"})) else: self.write(json.dumps({"response": "REJECTED"})) self.status_code(400, reason="invalid subscription data") self.finish()
def get(self): try: req_resp = stats.request(str(get_ip(self.request))) say("Received STATS request (" + req_resp + ")") except: error("Errored while handling request IP -- still served...") self.render("pages/stats.html", countries=stats.top_countries(), last_restart=stats.time(), devices=stats.unique_devices(), total_requests=stats.requests, requests_per_second=stats.requests_per_second())
def get_all_tweets(id): tweets = [] new_tweets = api.user_timeline(user_id=id, count=200) tweets.extend(new_tweets) oldest = tweets[-1].id - 1 while len(new_tweets) > 0: say("getting tweets before " + str(oldest)) new_tweets = api.user_timeline(user_id=id, count=200, max_id=oldest) tweets.extend(new_tweets) oldest = tweets[-1].id - 1 say("..." + str(len(tweets)) + " tweets downloaded so far") return [tweet._json for tweet in tweets ] # this makes me sad, but is there a better way? submit a PR.
def get(self): try: req_resp = stats.request(str(get_ip(self.request))) except: error("Errored while handling request IP -- still served...") say("Received INDEX request (" + req_resp + ")") flash = flashes.get_latest_flashes(1)[0] flash['text'] = tornado.escape.xhtml_unescape(flash['text']) time = str(flash['time']) if isinstance(flash['time'], basestring): time = arrow.Arrow.strptime( flash['time'], "%a %b %d %H:%M:%S +0000 %Y").humanize() elif isinstance(flash['time'], datetime.datetime): time = arrow.get(flash['time']).humanize() self.render("pages/index.html", flash=flash, time=time)
def _push_notification_to_subscribers(subscriptions, data): userio.say("Sending notification to " + str(subscriptions.count()) + " subscribers in a new thread...") for subscription in subscriptions: try: webpush( subscription_info=subscription, data=data, vapid_private_key=configuration. get_vapid_public_private_key_pair()[1], vapid_claims=VAPID_CLAIMS, #gcm_key=configuration.get_gcm_key(), ) except Exception as e: userio.error(" ...unable to send notification: " + str(e)) db.subscriptions.remove(subscription) userio.say(" ...removed subscription!") userio.ok("Finished sending notification.") thread.exit()
def operate(): ok("Beginning operation...") for account in databasehandler.configuration["monitored_accounts"]: say("Writing the account data for " + str(account)) databasehandler.write_account(account, twitterhandler.get_account_data(account)) say("Writing the other data for " + str(account)) scanner.update_account(account) # this operation is done at the beginning of operation because # it sets up the database on the first run, and it also grabs # any tweets quickly, before the first full pass is run on each account iteration = 0 while iteration < sys.maxint: if iteration % databasehandler.configuration["intervals"][ "fullpass"] == 0: for account in databasehandler.configuration["monitored_accounts"]: ok("Running full pass for " + str(account)) deleted = scanner.full_pass(account) if len(deleted) > 0: deleted_tweets = [ databasehandler.get_tweet(account, tweet) for tweet in deleted ] notificationhandler.notify_deletions(account, deleted) if iteration % databasehandler.configuration["intervals"][ "account"] == 0: for account in databasehandler.configuration["monitored_accounts"]: ok("Running account update...") databasehandler.write_account( account, twitterhandler.get_account_data(account)) for account in databasehandler.configuration["monitored_accounts"]: ok("Running partial update pass for " + str(account)) deleted = scanner.update_account(account) iteration += 1 time.sleep(databasehandler.configuration["intervals"]["pass"]) operate() # in 292471208677 years...
def get(self): try: req_resp = stats.request(str(get_ip(self.request))) except: error("Errored while handling request IP -- still served...") say("Received INDEX request (" + req_resp + ")") alerts = flashes.get_latest_flashes(3) for flash in alerts: flash['text'] = tornado.escape.xhtml_unescape(flash['text']) if isinstance(flash['time'], basestring) and "+0000" in flash['time']: flash['time'] = arrow.Arrow.strptime( flash['time'], "%a %b %d %H:%M:%S +0000 %Y").humanize() if isinstance(flash['time'], datetime.datetime): flash['time'] = arrow.get(flash['time']).humanize() self.render("pages/index.html", flashes=alerts, countries=stats.top_countries(), last_restart=stats.time(), devices=stats.unique_devices(), total_requests=stats.requests, requests_per_second=str(stats.requests_per_second())[:5])
def get(self): try: req_resp = stats.request(str(get_ip(self.request))) say("Received API request (" + req_resp + ")") except: error("Errored while handling request IP -- still served...") self.set_header("Content-Type", "application/json") latest = -1 try: latest = int(self.get_argument('latest')) except: pass # no latest flash specified data = { "server": "LibreNews Central", "channels": [k[2] for k in configuration.get_accounts()], "latest": [ flash for flash in flashes.get_latest_flashes(25) if int(flash['id']) > int(latest) ] } self.write( unicode(json.dumps(data, sort_keys=True, separators=(',', ':'))))
def load_flashes(): say("Loading latest flashes...") for accountPair in configuration.get_accounts(): say("Loading flashes from Twitter account " + accountPair[0] + " (" + accountPair[1] + ")") latest_tweets = twitter.get_latest_statuses(accountPair[0][1:]) for tweet in latest_tweets: url = None if "entities" in tweet: if "urls" in tweet['entities']: if len(tweet['entities']['urls']) > 0: url = tweet['entities']["urls"][-1]['expanded_url'] push_flash( generate_flash(tweet["text"], url, accountPair[1], tweet["id"], tweet["created_at"], accountPair[2]) ) say("Loaded " + str(len(latest_tweets)) + " flashes from " + accountPair[0]) ok("Loaded " + str(len(latest_flashes)) + " flashes")
def on_status(self, status): try: say("Detected flash...") if configuration.is_following(status.user.screen_name): say("...from a relevant source!") url = None if 'urls' in status.entities and len( status.entities['urls']) > 0: url = status.entities['urls'][0]["expanded_url"] flash = generate_flash( status.text, url, configuration.get_name(status.user.screen_name), status.id, status.created_at, configuration.get_channel(status.user.screen_name)) push_flash(flash) ok("Detected and pushed flash: " + str(flash)) else: say("...from an irrelevant source.") except Exception as e: error("Encountered an exception while processing a flash: " + str(e)) return True
import json import os from userio import say, error, ok, warn import sys # initialization if not os.path.isfile("configuration.json"): warn("No configuration file found. First launch?") say("Generating configuration file...") default_configuration = { "authentication": { "consumer_key": "consumerKeyHere", "consumer_secret": "consumerSecretHere", "access_token": "accessTokenHere", "access_token_secret": "accessTokenSecretHere" }, "monitored_accounts": [25073877, 822215679726100480], "intervals": { "fullpass": 600, "account": 1200, "pass": 5 }, "notifications": { "email": "*****@*****.**" } } with open("configuration.json", "w") as configuration_file: json.dump(default_configuration, configuration_file,
channels = None def get_channel(handle): global channels if channels is None: channels = {} for account in get_accounts(): channels[account[0]] = account[2] channels[account[0][1:]] = account[2] # so that @names defined in config still match return channels[handle] # Generate the default config. Will override existing config. def generate_config(): warn("Generating default config...") with open("config.json", "w") as config: json.dump(default_config, config, indent=4, sort_keys=True) ok("Generated default config!") if not os.path.exists("config.json"): warn("No configuration file found!") generate_config() say("Please edit the configuration file. LibreNews Server will now shutdown." ) sys.exit(1)
import tweepy import databasehandler as db import json from userio import say, ok, warn, error say("Establishing connection to Twitter...") auth = tweepy.OAuthHandler(db.get_authentication()["consumer_key"], db.get_authentication()["consumer_secret"]) auth.set_access_token(db.get_authentication()["access_token"], db.get_authentication()["access_token_secret"]) api = tweepy.API(auth) ok("Connection established!") #Twitter only allows access to a users most recent 3240 tweets with this method def get_all_tweets(id): tweets = [] new_tweets = api.user_timeline(user_id=id, count=200) tweets.extend(new_tweets) oldest = tweets[-1].id - 1 while len(new_tweets) > 0: say("getting tweets before " + str(oldest)) new_tweets = api.user_timeline(user_id=id, count=200, max_id=oldest) tweets.extend(new_tweets) oldest = tweets[-1].id - 1 say("..." + str(len(tweets)) + " tweets downloaded so far") return [tweet._json for tweet in tweets ] # this makes me sad, but is there a better way? submit a PR. def get_account_data(account):
def get_latest_statuses(username): say("Gathering latest statuses from " + username) return [status._json for status in api.user_timeline(username)]