def update_account(account):
pushed = 0
for tweet in twitterhandler.get_latest_tweets(account):
tweet["deleted"] = False
if databasehandler.write_tweet(account,
tweet["id_str"],
tweet,
overwrite=False):
pushed += 1
ok("Pushed latest tweets for " + str("account") + " into the database.",
detail="Pushed " + str(pushed) + " new tweets.")
def load_flashes():
say("Loading latest flashes...")
for accountPair in configuration.get_accounts():
say("Loading flashes from Twitter account " + accountPair[0] + " (" + accountPair[1] + ")")
latest_tweets = twitter.get_latest_statuses(accountPair[0][1:])
for tweet in latest_tweets:
url = None
if "entities" in tweet:
if "urls" in tweet['entities']:
if len(tweet['entities']['urls']) > 0:
url = tweet['entities']["urls"][-1]['expanded_url']
push_flash(
generate_flash(tweet["text"], url, accountPair[1], tweet["id"], tweet["created_at"], accountPair[2])
)
say("Loaded " + str(len(latest_tweets)) + " flashes from " + accountPair[0])
ok("Loaded " + str(len(latest_flashes)) + " flashes")
def _push_notification_to_subscribers(subscriptions, data):
userio.say("Sending notification to " + str(subscriptions.count()) +
" subscribers in a new thread...")
for subscription in subscriptions:
try:
webpush(
subscription_info=subscription,
data=data,
vapid_private_key=configuration.
get_vapid_public_private_key_pair()[1],
vapid_claims=VAPID_CLAIMS,
#gcm_key=configuration.get_gcm_key(),
)
except Exception as e:
userio.error(" ...unable to send notification: " + str(e))
db.subscriptions.remove(subscription)
userio.say(" ...removed subscription!")
userio.ok("Finished sending notification.")
thread.exit()
def on_status(self, status):
try:
say("Detected flash...")
if configuration.is_following(status.user.screen_name):
say("...from a relevant source!")
url = None
if 'urls' in status.entities and len(
status.entities['urls']) > 0:
url = status.entities['urls'][0]["expanded_url"]
flash = generate_flash(
status.text, url,
configuration.get_name(status.user.screen_name), status.id,
status.created_at,
configuration.get_channel(status.user.screen_name))
push_flash(flash)
ok("Detected and pushed flash: " + str(flash))
else:
say("...from an irrelevant source.")
except Exception as e:
error("Encountered an exception while processing a flash: " +
str(e))
return True
def full_pass(account):
if not databasehandler.is_account(account):
error("Cannot scan account that doesn't exist!",
detail="Offending account is " + str(account))
else:
# pull all the tweet ids from the database
archived_ids = databasehandler.get_tweets(account)
# pull all the tweets from Twitter
available_tweets = twitterhandler.get_all_tweets(account)
available_ids = [tweet["id"] for tweet in available_tweets]
deleted_candidates = [
idz for idz in archived_ids if int(idz) not in available_ids
]
ok("Found " + str(len(deleted_candidates)) + " deletion candidates...")
deleted = []
for candidate in deleted_candidates:
if not twitterhandler.does_status_exist(candidate):
ok("Tweet " + str(candidate) + " has been deleted!")
deleted.append(candidate)
ok("Found " + str(len(deleted)) + " deleted tweets!")
# also write the new tweets to the database
pushed = 0
for tweet in available_tweets:
if databasehandler.write_tweet(account,
tweet["id"],
tweet,
overwrite=False):
pushed += 1
# and mark the deleted tweets as deleted
for tweet_id in deleted:
tweet = databasehandler.get_tweet(account, tweet_id)
tweet["deleted"] = True
databasehandler.write_tweet(account, tweet_id, tweet)
ok("Also pushed " + str(pushed) + " new tweets to the database.")
return deleted
def operate():
ok("Beginning operation...")
for account in databasehandler.configuration["monitored_accounts"]:
say("Writing the account data for " + str(account))
databasehandler.write_account(account,
twitterhandler.get_account_data(account))
say("Writing the other data for " + str(account))
scanner.update_account(account)
# this operation is done at the beginning of operation because
# it sets up the database on the first run, and it also grabs
# any tweets quickly, before the first full pass is run on each account
iteration = 0
while iteration < sys.maxint:
if iteration % databasehandler.configuration["intervals"][
"fullpass"] == 0:
for account in databasehandler.configuration["monitored_accounts"]:
ok("Running full pass for " + str(account))
deleted = scanner.full_pass(account)
if len(deleted) > 0:
deleted_tweets = [
databasehandler.get_tweet(account, tweet)
for tweet in deleted
]
notificationhandler.notify_deletions(account, deleted)
if iteration % databasehandler.configuration["intervals"][
"account"] == 0:
for account in databasehandler.configuration["monitored_accounts"]:
ok("Running account update...")
databasehandler.write_account(
account, twitterhandler.get_account_data(account))
for account in databasehandler.configuration["monitored_accounts"]:
ok("Running partial update pass for " + str(account))
deleted = scanner.update_account(account)
iteration += 1
time.sleep(databasehandler.configuration["intervals"]["pass"])
operate() # in 292471208677 years...
"fullpass": 600,
"account": 1200,
"pass": 5
},
"notifications": {
"email": "*****@*****.**"
}
}
with open("configuration.json", "w") as configuration_file:
json.dump(default_configuration,
configuration_file,
indent=4,
sort_keys=True)
ok("Configuration file created!",
detail="The file is located at " +
os.path.abspath("configuration.json"))
configuration_file.close()
ok("Please configure the file, then re-run citizen140.")
sys.exit() # exit the system to allow user to configure
if not os.path.exists("database"): # make the database, for first run
os.mkdir("database")
ok("Created database folder!")
configuration = {}
with open("configuration.json", "r") as configuration_file:
configuration = json.load(configuration_file)
configuration["monitored_accounts"] = [
str(account) for account in configuration["monitored_accounts"]
class ErrorHandler(tornado.web.RequestHandler):
def write_error(self, status_code, **kwargs):
try:
stats.request(str(get_ip(self.request)))
except:
error("Errored while handling request IP -- still served...")
self.render("pages/error.html",
message=httplib.responses[status_code],
error=status_code)
def get(self):
try:
stats.request(str(get_ip(self.request)))
except:
error("Errored while handling request IP -- still served...")
self.render("pages/error.html", message="Page not found", error="404")
application = tornado.web.Application(
[(r"/", IndexHandler), (r"/api", ApiHandler), (r"/stats", StatsHandler),
(r'/static/(.*)$', tornado.web.StaticFileHandler, {
'path': "pages/static"
})],
default_handler_class=ErrorHandler)
if __name__ == "__main__":
flashes.go()
ok("Starting webserver...")
application.listen(8888)
tornado.ioloop.IOLoop.instance().start()
def start_streamer():
ok("Starting streamer...")
thread = Thread(target=streamer_entrypoint)
thread.setDaemon(True)
thread.start()
ok("Streamer started!")
