def createArticle(self, url, min_amount, threshold):
crawler = RedditCrawler()
article = crawler.getArticle(url)
gen = GenerateArticle()
gen.generate(article, threshold)
interface = ArticleCommit()
interface.commit(article, min_amount)
class TestFormater(unittest.TestCase):
@classmethod
def setUpClass(self):
self.crawler = RedditCrawler()
with open("./tests/data/cats_subreddit.txt") as reddit_file:
self.cats_subreddit = reddit_file.read()
with open("./tests/data/top_cats_threads.json") as top_threads_json:
self.top_cats_threads = json.load(top_threads_json)
@patch.object(RedditCrawler, 'request_page_text')
def test_create_top_threads(self, mock_request_page):
mock_request_page.return_value = self.cats_subreddit
top_threads = self.crawler.get_top_threads("cats")
top_threads_list = self.crawler.create_top_threads_list(top_threads)
self.assertEqual(top_threads_list, self.top_cats_threads)
def setUpClass(self):
self.crawler = RedditCrawler()
with open("./tests/data/cats_subreddit.txt") as reddit_file:
self.cats_subreddit = reddit_file.read()
with open("./tests/data/top_cats_threads.json") as top_threads_json:
self.top_cats_threads = json.load(top_threads_json)
def monthly_run(self, limit, min_amount, threshold):
crawler = RedditCrawler()
monthlyTopURLs = crawler.getURLMonthlyTop(limit)
for url in monthlyTopURLs :
self.createArticle(url, min_amount, threshold)
def main(self):
red = RedditCrawler()
word = raw_input('Enter a search term: ')
red.search(word)
""+ Statistics.get_reddit_error_sites() + "\n" \
"Facebook Errorlog: \n" \
"" + Statistics.get_facebook_error_sites() + "\n"
Statistics.reset_facebook()
Statistics.reset_twitter()
Statistics.reset_telegram()
Statistics.reset_reddit()
filename = time.strftime("%d-%m-%Y - Daily Social Media Report.txt")
save_to_file(output, filename)
print output
twitter = Twitter_Crawler()
telegram = TelegramMemberCrawler()
reddit = RedditCrawler()
facebook = FacebookCrawler()
schedule.every(30).minutes.do(ping, 0)
schedule.every(180).seconds.do(twitter.get_twitter_feeds)
schedule.every(180).seconds.do(reddit.get_reddit_posts)
schedule.every(180).seconds.do(facebook.get_facebook_posts)
schedule.every().day.at("00:01").do(twitter.get_description_information)
schedule.every().day.at("00:01").do(telegram.CrawlMembers)
schedule.every().day.at("00:01").do(reddit.get_reddit_general)
schedule.every().day.at("00:01").do(facebook.get_description_information)
schedule.every().day.at("00:11").do(create_report)
while True:
schedule.run_pending()
time.sleep(1)
import config as c
from nltk.sentiment.vader import SentimentIntensityAnalyzer
from socialMediaObjectCreator import socialMedia
from Database import Database
from RedditCrawler import RedditCrawler
from TwitterCrawler import TwitterCrawler
from GUI import GUI
print("Start")
db = Database(c.host, c.user, c.password, c.database)
db.createTable()
reddit = RedditCrawler("Crimes", c.reddit_id, c.reddit_secret)
twitter = TwitterCrawler(c.consumer_key, c.consumer_secret, c.access_token,
c.access_token_secret)
ui = GUI(db, reddit, twitter)
ui.root.mainloop()
db.disconnect()
import sys
import json
from telegram.ext import Updater, CommandHandler
import logging
logging.basicConfig(
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
level=logging.INFO)
sys.path.append("./commom")
sys.path.append("./reddit")
from RedditCrawler import RedditCrawler
import defines
crawler = RedditCrawler()
def prepare_message(list_of_subreddits):
message = ""
for subreddit in list_of_subreddits:
top_threads = crawler.get_top_threads(subreddit)
top_threads_list = crawler.create_top_threads_list(top_threads)
message += "--------------------------------------\n"
message += "TOP THREADS FOR {} SUBREDDIT\n".format(subreddit)
if top_threads_list:
for thread in top_threads_list:
message += "\n"
