def test_no_previous_scraping_sessions_doesnt_do_anything(self, ScrapingSession, get_days_between, DailySummary):
        previous_session = None
        ScrapingSession.get_previous_session.return_value = previous_session

        background.update_daily_summaries()

        DailySummary.create.has_calls([])
    def test_uses_job_listings_to_create_daily_summary(self):

        job1 = JobListing.objects.create(
            jobid=1,
            title='title1',
            description='description1',
            added=datetime.datetime(year=2015, month=9, day=1).date())
        
        job2 = JobListing.objects.create(
            jobid=2,
            title='title2',
            description='description2',
        added=datetime.datetime(year=2015, month=9, day=2).date())
        
        job3 = JobListing.objects.create(
            jobid=3,
            title='title3',
            description='description3',
        added=datetime.datetime(year=2015, month=9, day=2).date())

        session = scraping.models.Session.objects.create(datetime=datetime.datetime(year=2015, month=9, day=3))

        background.update_daily_summaries()

        daily_summary1 = DailySummary.objects.first()
        daily_summary2 = DailySummary.objects.last()
    
        self.assertEqual(daily_summary1.date, job1.added)
        self.assertEqual(daily_summary2.date, job2.added)
        self.assertEqual(daily_summary1.n_posts, 1)
        self.assertEqual(daily_summary2.n_posts, 2)
    def test_last_summary_is_none_gets_the_first_job_listing(self, JobListing, ScrapingSession, get_days_between, DailySummary):
        last_summary = None
        DailySummary.get_last_summary.return_value = last_summary

        background.update_daily_summaries()

        JobListing.get_earliest_job_listing.assert_called_once_with()
    def test_creates_a_new_daily_summary_for_each_date(self, ScrapingSession, get_days_between, DailySummary):
        dates = [mock.Mock(), mock.Mock()]
        get_days_between.return_value = dates

        background.update_daily_summaries()

        DailySummary.create.has_calls([mock.call(date) for date in dates])
    def test_gets_the_days_between_the_last_scraping_session_and_the_last_daily_summary(self, ScrapingSession, get_days_between, DailySummary):
        previous_scraping_session = mock.Mock()
        ScrapingSession.get_previous_session.return_value = previous_scraping_session
        last_summary = mock.Mock()
        DailySummary.get_last_summary.return_value = last_summary

        background.update_daily_summaries()

        get_days_between.assert_called_with(last_summary.date, previous_scraping_session.datetime.date())
    def test_daily_summaries_are_saved(self, ScrapingSession, get_days_between, DailySummary):
        dates = [mock.Mock(), mock.Mock()]
        get_days_between.return_value = dates
        daily_summaries = [mock.Mock(), mock.Mock()]
        DailySummary.create.side_effect = daily_summaries

        background.update_daily_summaries()

        for daily_summary in daily_summaries:
            daily_summary.save.assert_called_once_with()
    def test_last_summary_is_none_uses_first_job_listing_as_start_date(self, JobListing, ScrapingSession, get_days_between, DailySummary):
        '''
        Actually uses the day before the job added date
        '''
        last_summary = None
        DailySummary.get_last_summary.return_value = last_summary
        session = mock.Mock()
        ScrapingSession.get_previous_session.return_value = session
        earliest_job = JobListing.get_earliest_job_listing()
        day_before = earliest_job.added - datetime.timedelta(days=1)

        background.update_daily_summaries()

        get_days_between.assert_called_once_with(day_before, session.datetime.date())
    def handle(self, *args, **options):

        logger.setLevel(logging.INFO)
        handler = logging.StreamHandler()
        handler.setLevel(logging.INFO)
        logger.addHandler(handler)
        formatter = logging.Formatter("%(levelname)s: %(name)s: %(message)s")
        handler.setFormatter(formatter)
        bg_logger.setLevel(logging.INFO)
        bg_logger.addHandler(handler)
        # logger.propagate = False
        # file_handler = logging.FileHandler(options['log'])
        # file_handler.setLevel(logging.INFO)
        # file_handler.setFormatter(formatter)
        # logger.addHandler(file_handler)
        # bg_logger.addHandler(file_handler)

        logger.info("Start BG data aggregation process")

        while True:
            update_daily_summaries()
            time.sleep(86400.0)
    def test_gets_the_last_daily_summary(self, ScrapingSession, get_days_between, DailySummary):
        background.update_daily_summaries()

        DailySummary.get_last_summary.assert_called_once_with()
示例#10
0
    def test_gets_the_date_of_the_last_scraping_session(self, ScrapingSession, get_days_between, DailySummary):
        background.update_daily_summaries()

        ScrapingSession.get_previous_session.assert_called_once_with()