def update_daily_summaries(): logger.info('Update Daily Summaries') previous_session = ScrapingSession.get_previous_session() logger.info('Previous Scraping Session: {}'.format(previous_session.datetime if previous_session else None)) if previous_session == None: return last_summary = DailySummary.get_last_summary() logger.info('Last Daily Summary: {}'.format(last_summary.date if last_summary else None)) if last_summary == None: start_date = JobListing.get_earliest_job_listing().added - datetime.timedelta(days=1) else: start_date = last_summary.date dates_between = data_science_jobs.get_days_between( start_date, previous_session.datetime.date()) if len(dates_between): logger.info('Getting Daily Summaries Between: {} - {}'.format(dates_between[0], dates_between[-1])) else: logger.info('Daily Summaries Up-To-Date') for date in dates_between: daily_summary = DailySummary.create(date=date) logger.info('{}: n_posts: {}'.format(date, daily_summary.n_posts)) daily_summary.save()
def update_monthly_summaries(): logger.info('Update Monthly Summaries') previous_session = ScrapingSession.get_previous_session() logger.info('Previous Scraping Session: {}'.format(previous_session.datetime if previous_session else None)) if previous_session == None: return last_summary = MonthlySummary.get_last_summary() logger.info('Last Monthly Summary: {}'.format(last_summary.date if last_summary else last_summary)) if last_summary == None: start_date = JobListing.get_earliest_job_listing().added start_date = start_date - datetime.timedelta(days=start_date.day) start_date = datetime.datetime(year=start_date.year, month=start_date.month, day=1).date() else: start_date = last_summary.date previous_session_month = previous_session.datetime.date() previous_session_month = previous_session_month - datetime.timedelta(days=previous_session_month.day - 1) months_between = data_science_jobs.get_months_between( start_date, previous_session.datetime.date()) logger.info('Months between: {}'.format(months_between)) for date in months_between: monthly_summary = MonthlySummary.create(date=date) logger.info('{}: n_posts: {}'.format(date, monthly_summary.n_posts)) monthly_summary.save()
def test_returns_previous_session(self): session = Session.objects.create(datetime=datetime.datetime(year=2015, month=1, day=1)) output = Session.get_previous_session() self.assertEqual(output, session)
def test_no_previous_session_returns_none(self): session = Session.get_previous_session() self.assertEqual(session, None)