def main():
minute = datetime.datetime.now().minute
five_min_interval = (int(minute) / 5) - 1
one_digit_minute = int(str(minute)[-1])
print("################################")
print("#### OPSEC Search Bootstrap ####")
print("################################")
print("----- User Specific Search -----")
print("Attempting site/user specific search")
# Twitter
try:
user = twitter.get_users()[one_digit_minute]
twitter.get_user_tweets(user)
except IndexError:
print("No Twitter user found at index " + str(one_digit_minute))
# Reddit
try:
author = reddit.get_users()[one_digit_minute]
reddit.get_user_comments(author)
except IndexError:
print("No Reddit user found at index " + str(one_digit_minute))
# StackExchange
try:
account_id = stackexchange.get_users()[one_digit_minute]
stackexchange.get_user_posts(account_id)
except IndexError:
print("No StackExchange user found at index " + str(one_digit_minute))
print("-------- General search --------")
if (minute % 5) == 0:
print("Attempting general site search...")
try:
keyword = opsecHeader.get_user_keywords('all',
'twitter')[five_min_interval]
twitter.search_twitter(keyword)
except IndexError:
print("No twitter keyword at index " + str(five_min_interval))
try:
keyword = opsecHeader.get_user_keywords('all',
'facebook')[five_min_interval]
facebook.search_facebook(keyword)
except IndexError:
print("No facebook keyword at index " + str(five_min_interval))
try:
keyword = opsecHeader.get_user_keywords('all',
'wordpress')[five_min_interval]
wordpress.search_wordpress(keyword)
except IndexError:
print("No wordpress keyword at index " + str(five_min_interval))
else:
print("Minute not a multiple of 5, not attempting general site search...")
def _main():
config = get_config()
pool = ThreadPool(4)
connection = Connection(config.get("mongodb", "host"),
config.getint("mongodb", "port"))
db = connection[config.get("mongodb", "db")]
searches = db.twitter_searches
tweets = db.twitter_tweets
for keyword in config.get("twitter", "keywords").split(":"):
last_searches = searches.find({"keyword": keyword})
if last_searches.count():
since_id = last_searches.sort("created_at",
DESCENDING).limit(1)[0]["max_id"]
else:
since_id = 0
response = search_twitter(config, keyword, since_id=str(since_id))
new_search = subdict(response, ["max_id", "since_id"])
new_search["keyword"] = keyword
new_search["created_at"] = datetime.now()
searches.insert(new_search)
print "Found %d results on %s" % (len(response["results"]), keyword)
for result in response["results"]:
if result["from_user"] == "no404bot": continue
urls_list = result["entities"].get("urls", None)
if not urls_list: continue
tweet = subdict(result, [
"from_user", "from_user_id", "text", "iso_language_code", "id"
])
tweet["created_at"] = datetime.strptime(result["created_at"],
TWITTER_DATETIME_FORMAT)
tweet["urls"] = urls_list
tweets.insert(tweet)
pool.add_task(read_tweet, pool, db, config, tweet["id"])
pool.wait_completion()
def _main():
config = get_config()
pool = ThreadPool(4)
connection = Connection(config.get("mongodb", "host"), config.getint("mongodb", "port"))
db = connection[config.get("mongodb", "db")]
searches = db.twitter_searches
tweets = db.twitter_tweets
for keyword in config.get("twitter", "keywords").split(":"):
last_searches = searches.find({"keyword":keyword})
if last_searches.count():
since_id = last_searches.sort("created_at", DESCENDING).limit(1)[0]["max_id"]
else:
since_id = 0
response = search_twitter(config, keyword, since_id=str(since_id))
new_search = subdict(response, ["max_id", "since_id"])
new_search["keyword"] = keyword
new_search["created_at"] = datetime.now()
searches.insert(new_search)
print "Found %d results on %s" % (len(response["results"]), keyword)
for result in response["results"]:
if result["from_user"] == "no404bot": continue
urls_list = result["entities"].get("urls", None)
if not urls_list: continue
tweet = subdict(result, ["from_user", "from_user_id", "text", "iso_language_code", "id"])
tweet["created_at"] = datetime.strptime(result["created_at"], TWITTER_DATETIME_FORMAT)
tweet["urls"] = urls_list
tweets.insert(tweet)
pool.add_task(read_tweet, pool, db, config, tweet["id"])
pool.wait_completion()
