def post(self):
say("Receiving new subscription: " + self.request.body)
self.set_header("Content-Type", "application/json")
if push.register_new_receiver(
tornado.escape.json_decode(self.request.body)):
self.write(json.dumps({"response": "OK"}))
else:
self.write(json.dumps({"response": "REJECTED"}))
self.status_code(400, reason="invalid subscription data")
self.finish()
def get(self):
try:
req_resp = stats.request(str(get_ip(self.request)))
say("Received STATS request (" + req_resp + ")")
except:
error("Errored while handling request IP -- still served...")
self.render("pages/stats.html",
countries=stats.top_countries(),
last_restart=stats.time(),
devices=stats.unique_devices(),
total_requests=stats.requests,
requests_per_second=stats.requests_per_second())
def get_all_tweets(id):
tweets = []
new_tweets = api.user_timeline(user_id=id, count=200)
tweets.extend(new_tweets)
oldest = tweets[-1].id - 1
while len(new_tweets) > 0:
say("getting tweets before " + str(oldest))
new_tweets = api.user_timeline(user_id=id, count=200, max_id=oldest)
tweets.extend(new_tweets)
oldest = tweets[-1].id - 1
say("..." + str(len(tweets)) + " tweets downloaded so far")
return [tweet._json for tweet in tweets
] # this makes me sad, but is there a better way? submit a PR.
def get(self):
try:
req_resp = stats.request(str(get_ip(self.request)))
except:
error("Errored while handling request IP -- still served...")
say("Received INDEX request (" + req_resp + ")")
flash = flashes.get_latest_flashes(1)[0]
flash['text'] = tornado.escape.xhtml_unescape(flash['text'])
time = str(flash['time'])
if isinstance(flash['time'], basestring):
time = arrow.Arrow.strptime(
flash['time'], "%a %b %d %H:%M:%S +0000 %Y").humanize()
elif isinstance(flash['time'], datetime.datetime):
time = arrow.get(flash['time']).humanize()
self.render("pages/index.html", flash=flash, time=time)
def _push_notification_to_subscribers(subscriptions, data):
userio.say("Sending notification to " + str(subscriptions.count()) +
" subscribers in a new thread...")
for subscription in subscriptions:
try:
webpush(
subscription_info=subscription,
data=data,
vapid_private_key=configuration.
get_vapid_public_private_key_pair()[1],
vapid_claims=VAPID_CLAIMS,
#gcm_key=configuration.get_gcm_key(),
)
except Exception as e:
userio.error(" ...unable to send notification: " + str(e))
db.subscriptions.remove(subscription)
userio.say(" ...removed subscription!")
userio.ok("Finished sending notification.")
thread.exit()
def operate():
ok("Beginning operation...")
for account in databasehandler.configuration["monitored_accounts"]:
say("Writing the account data for " + str(account))
databasehandler.write_account(account,
twitterhandler.get_account_data(account))
say("Writing the other data for " + str(account))
scanner.update_account(account)
# this operation is done at the beginning of operation because
# it sets up the database on the first run, and it also grabs
# any tweets quickly, before the first full pass is run on each account
iteration = 0
while iteration < sys.maxint:
if iteration % databasehandler.configuration["intervals"][
"fullpass"] == 0:
for account in databasehandler.configuration["monitored_accounts"]:
ok("Running full pass for " + str(account))
deleted = scanner.full_pass(account)
if len(deleted) > 0:
deleted_tweets = [
databasehandler.get_tweet(account, tweet)
for tweet in deleted
]
notificationhandler.notify_deletions(account, deleted)
if iteration % databasehandler.configuration["intervals"][
"account"] == 0:
for account in databasehandler.configuration["monitored_accounts"]:
ok("Running account update...")
databasehandler.write_account(
account, twitterhandler.get_account_data(account))
for account in databasehandler.configuration["monitored_accounts"]:
ok("Running partial update pass for " + str(account))
deleted = scanner.update_account(account)
iteration += 1
time.sleep(databasehandler.configuration["intervals"]["pass"])
operate() # in 292471208677 years...
def get(self):
try:
req_resp = stats.request(str(get_ip(self.request)))
except:
error("Errored while handling request IP -- still served...")
say("Received INDEX request (" + req_resp + ")")
alerts = flashes.get_latest_flashes(3)
for flash in alerts:
flash['text'] = tornado.escape.xhtml_unescape(flash['text'])
if isinstance(flash['time'],
basestring) and "+0000" in flash['time']:
flash['time'] = arrow.Arrow.strptime(
flash['time'], "%a %b %d %H:%M:%S +0000 %Y").humanize()
if isinstance(flash['time'], datetime.datetime):
flash['time'] = arrow.get(flash['time']).humanize()
self.render("pages/index.html",
flashes=alerts,
countries=stats.top_countries(),
last_restart=stats.time(),
devices=stats.unique_devices(),
total_requests=stats.requests,
requests_per_second=str(stats.requests_per_second())[:5])
def get(self):
try:
req_resp = stats.request(str(get_ip(self.request)))
say("Received API request (" + req_resp + ")")
except:
error("Errored while handling request IP -- still served...")
self.set_header("Content-Type", "application/json")
latest = -1
try:
latest = int(self.get_argument('latest'))
except:
pass # no latest flash specified
data = {
"server":
"LibreNews Central",
"channels": [k[2] for k in configuration.get_accounts()],
"latest": [
flash for flash in flashes.get_latest_flashes(25)
if int(flash['id']) > int(latest)
]
}
self.write(
unicode(json.dumps(data, sort_keys=True, separators=(',', ':'))))
def load_flashes():
say("Loading latest flashes...")
for accountPair in configuration.get_accounts():
say("Loading flashes from Twitter account " + accountPair[0] + " (" + accountPair[1] + ")")
latest_tweets = twitter.get_latest_statuses(accountPair[0][1:])
for tweet in latest_tweets:
url = None
if "entities" in tweet:
if "urls" in tweet['entities']:
if len(tweet['entities']['urls']) > 0:
url = tweet['entities']["urls"][-1]['expanded_url']
push_flash(
generate_flash(tweet["text"], url, accountPair[1], tweet["id"], tweet["created_at"], accountPair[2])
)
say("Loaded " + str(len(latest_tweets)) + " flashes from " + accountPair[0])
ok("Loaded " + str(len(latest_flashes)) + " flashes")
def on_status(self, status):
try:
say("Detected flash...")
if configuration.is_following(status.user.screen_name):
say("...from a relevant source!")
url = None
if 'urls' in status.entities and len(
status.entities['urls']) > 0:
url = status.entities['urls'][0]["expanded_url"]
flash = generate_flash(
status.text, url,
configuration.get_name(status.user.screen_name), status.id,
status.created_at,
configuration.get_channel(status.user.screen_name))
push_flash(flash)
ok("Detected and pushed flash: " + str(flash))
else:
say("...from an irrelevant source.")
except Exception as e:
error("Encountered an exception while processing a flash: " +
str(e))
return True
import json
import os
from userio import say, error, ok, warn
import sys
# initialization
if not os.path.isfile("configuration.json"):
warn("No configuration file found. First launch?")
say("Generating configuration file...")
default_configuration = {
"authentication": {
"consumer_key": "consumerKeyHere",
"consumer_secret": "consumerSecretHere",
"access_token": "accessTokenHere",
"access_token_secret": "accessTokenSecretHere"
},
"monitored_accounts": [25073877, 822215679726100480],
"intervals": {
"fullpass": 600,
"account": 1200,
"pass": 5
},
"notifications": {
"email": "*****@*****.**"
}
}
with open("configuration.json", "w") as configuration_file:
json.dump(default_configuration,
configuration_file,
channels = None
def get_channel(handle):
global channels
if channels is None:
channels = {}
for account in get_accounts():
channels[account[0]] = account[2]
channels[account[0][1:]] = account[2]
# so that @names defined in config still match
return channels[handle]
# Generate the default config. Will override existing config.
def generate_config():
warn("Generating default config...")
with open("config.json", "w") as config:
json.dump(default_config, config, indent=4, sort_keys=True)
ok("Generated default config!")
if not os.path.exists("config.json"):
warn("No configuration file found!")
generate_config()
say("Please edit the configuration file. LibreNews Server will now shutdown."
)
sys.exit(1)
import tweepy
import databasehandler as db
import json
from userio import say, ok, warn, error
say("Establishing connection to Twitter...")
auth = tweepy.OAuthHandler(db.get_authentication()["consumer_key"],
db.get_authentication()["consumer_secret"])
auth.set_access_token(db.get_authentication()["access_token"],
db.get_authentication()["access_token_secret"])
api = tweepy.API(auth)
ok("Connection established!")
#Twitter only allows access to a users most recent 3240 tweets with this method
def get_all_tweets(id):
tweets = []
new_tweets = api.user_timeline(user_id=id, count=200)
tweets.extend(new_tweets)
oldest = tweets[-1].id - 1
while len(new_tweets) > 0:
say("getting tweets before " + str(oldest))
new_tweets = api.user_timeline(user_id=id, count=200, max_id=oldest)
tweets.extend(new_tweets)
oldest = tweets[-1].id - 1
say("..." + str(len(tweets)) + " tweets downloaded so far")
return [tweet._json for tweet in tweets
] # this makes me sad, but is there a better way? submit a PR.
def get_account_data(account):
def get_latest_statuses(username):
say("Gathering latest statuses from " + username)
return [status._json for status in api.user_timeline(username)]