def test_returns_ealiest_job(self):
job1 = JobListing.objects.create(jobid=1, title='Job 1', added=datetime.datetime(year=2015, month=9, day=1).date())
job2 = JobListing.objects.create(jobid=2, title='Job 2', added=datetime.datetime(year=2015, month=8, day=1).date())
job3 = JobListing.objects.create(jobid=3, title='Job 3', added=datetime.datetime(year=2015, month=10, day=1).date())
job_listing = JobListing.get_earliest_job_listing()
self.assertEqual(job_listing, job2)
def test_get_n_posts_returns_n_job_listings_for_m_month(self):
job1 = JobListing.objects.create(jobid=1,
title='Title 1',
description='Description 1',
added=datetime.datetime(2015, 8, 1).date())
job2 = JobListing.objects.create(jobid=2,
title='Title 2',
description='Description 2',
added=datetime.datetime(2015, 9, 2).date())
job3 = JobListing.objects.create(jobid=3,
title='Title 3',
description='Description 3',
added=datetime.datetime(2015, 9, 2).date())
n_posts = JobListing.get_n_posts(datetime.datetime(2015, 9, 1).date(), month=True)
self.assertEqual(n_posts, 2)
n_posts = JobListing.get_n_posts(datetime.datetime(2015, 8, 1).date(), month=True)
self.assertEqual(n_posts, 1)
def test_last_summary_is_none_uses_first_job_listing_as_start_date(self, JobListing, ScrapingSession, get_days_between, DailySummary):
'''
Actually uses the day before the job added date
'''
last_summary = None
DailySummary.get_last_summary.return_value = last_summary
session = mock.Mock()
ScrapingSession.get_previous_session.return_value = session
earliest_job = JobListing.get_earliest_job_listing()
day_before = earliest_job.added - datetime.timedelta(days=1)
background.update_daily_summaries()
get_days_between.assert_called_once_with(day_before, session.datetime.date())
def update_daily_summaries():
logger.info('Update Daily Summaries')
previous_session = ScrapingSession.get_previous_session()
logger.info('Previous Scraping Session: {}'.format(previous_session.datetime if previous_session else None))
if previous_session == None:
return
last_summary = DailySummary.get_last_summary()
logger.info('Last Daily Summary: {}'.format(last_summary.date if last_summary else None))
if last_summary == None:
start_date = JobListing.get_earliest_job_listing().added - datetime.timedelta(days=1)
else:
start_date = last_summary.date
dates_between = data_science_jobs.get_days_between(
start_date,
previous_session.datetime.date())
if len(dates_between):
logger.info('Getting Daily Summaries Between: {} - {}'.format(dates_between[0], dates_between[-1]))
else:
logger.info('Daily Summaries Up-To-Date')
for date in dates_between:
daily_summary = DailySummary.create(date=date)
logger.info('{}: n_posts: {}'.format(date, daily_summary.n_posts))
daily_summary.save()
def update_monthly_summaries():
logger.info('Update Monthly Summaries')
previous_session = ScrapingSession.get_previous_session()
logger.info('Previous Scraping Session: {}'.format(previous_session.datetime if previous_session else None))
if previous_session == None:
return
last_summary = MonthlySummary.get_last_summary()
logger.info('Last Monthly Summary: {}'.format(last_summary.date if last_summary else last_summary))
if last_summary == None:
start_date = JobListing.get_earliest_job_listing().added
start_date = start_date - datetime.timedelta(days=start_date.day)
start_date = datetime.datetime(year=start_date.year, month=start_date.month, day=1).date()
else:
start_date = last_summary.date
previous_session_month = previous_session.datetime.date()
previous_session_month = previous_session_month - datetime.timedelta(days=previous_session_month.day - 1)
months_between = data_science_jobs.get_months_between(
start_date,
previous_session.datetime.date())
logger.info('Months between: {}'.format(months_between))
for date in months_between:
monthly_summary = MonthlySummary.create(date=date)
logger.info('{}: n_posts: {}'.format(date, monthly_summary.n_posts))
monthly_summary.save()
def test_returns_none_if_no_job_listings(self):
job_listing = JobListing.get_earliest_job_listing()
self.assertEqual(job_listing, None)
def create(cls, date):
n_posts = JobListing.get_n_posts(date)
daily_summary = cls(date=date, n_posts=n_posts)
return daily_summary
def create(cls, date):
n_posts = JobListing.get_n_posts(date, month=True)
monthly_summary = cls(date=date, n_posts=n_posts)
return monthly_summary
