def setUp(self): self.scraper = FacebookScraper() self.posts = self.scraper.scrape_facebook( profile_filename='tests/snapshots/fb-profile-04-20-2015.json', posts_filename='tests/snapshots/fb-posts-04-20-2015.json') self.insights = self.scraper.scrape_post_insights( self.posts[0], filename='tests/snapshots/fb-insights-04-20-2015.json')
def scrape_facebook(): """ Scrape Facebook """ db = dataset.connect(app_config.POSTGRES_URL) scraper = FacebookScraper() posts = scraper.scrape_facebook() insights = scraper.scrape_insights(posts) scraper.write(db, posts, insights)
def get_or_create(self): user_list = self.filter_by(username=self._dict_data['username']) if not user_list: term = self._dict_data['username'] scraper = FacebookScraper() scraper.search_profile_by_term(term) scraper = TwitterScraper() scraper.search_profile_by_term(term) user_list = self.filter_by(username=self._dict_data['username']) return user_list
def test_search_profiles_by_term(self): self.assertEquals(0, User.objects.filter_by().count()) term = 'Jhon Smith' scraper = FacebookScraper() scraper.search_profile_by_term(term) self.assertNotEquals(0, User.objects.filter_by().count())
def create_users(self): term = 'Jhon Smith' scraper = FacebookScraper() scraper.search_profile_by_term(term) scraper = TwitterScraper() scraper.search_profile_by_term(term)