def createArticle(self, url, min_amount, threshold): crawler = RedditCrawler() article = crawler.getArticle(url) gen = GenerateArticle() gen.generate(article, threshold) interface = ArticleCommit() interface.commit(article, min_amount)
class TestFormater(unittest.TestCase): @classmethod def setUpClass(self): self.crawler = RedditCrawler() with open("./tests/data/cats_subreddit.txt") as reddit_file: self.cats_subreddit = reddit_file.read() with open("./tests/data/top_cats_threads.json") as top_threads_json: self.top_cats_threads = json.load(top_threads_json) @patch.object(RedditCrawler, 'request_page_text') def test_create_top_threads(self, mock_request_page): mock_request_page.return_value = self.cats_subreddit top_threads = self.crawler.get_top_threads("cats") top_threads_list = self.crawler.create_top_threads_list(top_threads) self.assertEqual(top_threads_list, self.top_cats_threads)
def setUpClass(self): self.crawler = RedditCrawler() with open("./tests/data/cats_subreddit.txt") as reddit_file: self.cats_subreddit = reddit_file.read() with open("./tests/data/top_cats_threads.json") as top_threads_json: self.top_cats_threads = json.load(top_threads_json)
def monthly_run(self, limit, min_amount, threshold): crawler = RedditCrawler() monthlyTopURLs = crawler.getURLMonthlyTop(limit) for url in monthlyTopURLs : self.createArticle(url, min_amount, threshold)
def main(self): red = RedditCrawler() word = raw_input('Enter a search term: ') red.search(word)
""+ Statistics.get_reddit_error_sites() + "\n" \ "Facebook Errorlog: \n" \ "" + Statistics.get_facebook_error_sites() + "\n" Statistics.reset_facebook() Statistics.reset_twitter() Statistics.reset_telegram() Statistics.reset_reddit() filename = time.strftime("%d-%m-%Y - Daily Social Media Report.txt") save_to_file(output, filename) print output twitter = Twitter_Crawler() telegram = TelegramMemberCrawler() reddit = RedditCrawler() facebook = FacebookCrawler() schedule.every(30).minutes.do(ping, 0) schedule.every(180).seconds.do(twitter.get_twitter_feeds) schedule.every(180).seconds.do(reddit.get_reddit_posts) schedule.every(180).seconds.do(facebook.get_facebook_posts) schedule.every().day.at("00:01").do(twitter.get_description_information) schedule.every().day.at("00:01").do(telegram.CrawlMembers) schedule.every().day.at("00:01").do(reddit.get_reddit_general) schedule.every().day.at("00:01").do(facebook.get_description_information) schedule.every().day.at("00:11").do(create_report) while True: schedule.run_pending() time.sleep(1)
import config as c from nltk.sentiment.vader import SentimentIntensityAnalyzer from socialMediaObjectCreator import socialMedia from Database import Database from RedditCrawler import RedditCrawler from TwitterCrawler import TwitterCrawler from GUI import GUI print("Start") db = Database(c.host, c.user, c.password, c.database) db.createTable() reddit = RedditCrawler("Crimes", c.reddit_id, c.reddit_secret) twitter = TwitterCrawler(c.consumer_key, c.consumer_secret, c.access_token, c.access_token_secret) ui = GUI(db, reddit, twitter) ui.root.mainloop() db.disconnect()
import sys import json from telegram.ext import Updater, CommandHandler import logging logging.basicConfig( format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', level=logging.INFO) sys.path.append("./commom") sys.path.append("./reddit") from RedditCrawler import RedditCrawler import defines crawler = RedditCrawler() def prepare_message(list_of_subreddits): message = "" for subreddit in list_of_subreddits: top_threads = crawler.get_top_threads(subreddit) top_threads_list = crawler.create_top_threads_list(top_threads) message += "--------------------------------------\n" message += "TOP THREADS FOR {} SUBREDDIT\n".format(subreddit) if top_threads_list: for thread in top_threads_list: message += "\n"