def test_get_google_batch(self):
"""
Scenario: Get a generator that returns the list of Google terms and date ranges to be
collected from the API organised in batches of up to 30 terms as per API documentation
Given a FluModel with an id value of 1 exists
And a list of 31 terms for such FluModel id
When querying for a list of missing date ranges
Then the generator returns items grouped by term batch and then date range
"""
with self.app.app_context():
terms_expected = []
for idx in range(31):
flu_model_google_term = FluModelGoogleTerm()
flu_model_google_term.flu_model_id = 1
flu_model_google_term.google_term_id = idx
flu_model_google_term.save()
google_term = GoogleTerm()
google_term.id = idx
google_term.term = 'Term %d' % idx
terms_expected.append('Term %d' % idx)
google_term.save()
missing_dates = [(date(2018, 1, 1), date(2018, 1, 1)),
(date(2018, 1, 6), date(2018, 1, 9)),
(date(2018, 1, 11), date(2018, 1, 15))]
result = list(get_google_batch(1, missing_dates))
terms_grouped = (terms_expected[0:30], terms_expected[30:31])
counter = 0
for terms_grouped_item in terms_grouped:
for missing_dates_tuple in missing_dates:
res_terms, res_start, res_end = result[counter]
self.assertListEqual(res_terms, terms_grouped_item)
self.assertEqual(res_start, missing_dates_tuple[0])
self.assertEqual(res_end, missing_dates_tuple[1])
counter += 1
def test_runsched_fetch_google_scores(self, mock_verify, mock_client):
"""
Evaluate parameters passed to fetch_google_scores and set_and_verify_google_dates
functions in _run_sched_for_model_no_set_dates
"""
last_date = date.today() - timedelta(days=4)
end_date = date.today() - timedelta(days=3)
with self.app.app_context():
google_date = GoogleDate(1, last_date)
google_date.save()
model_score = ModelScore()
model_score.flu_model_id = 1
model_score.score_date = end_date
model_score.score_value = 0.5
model_score.region = 'e'
model_score.save()
google_term = GoogleTerm()
google_term.id = 1
google_term.term = 'Term 1'
google_term.save()
flu_model_google_term = FluModelGoogleTerm()
flu_model_google_term.flu_model_id = 1
flu_model_google_term.google_term_id = 1
flu_model_google_term.save()
patched_api_client = mock_client.return_value
score_calculator.runsched([1], self.app)
patched_api_client.fetch_google_scores.assert_called_with(
['Term 1'], last_date, end_date)
mock_verify.assert_called_with(1, [end_date])
def test_set_verify_google_dates(self):
"""
Scenario: Persists the date of a complete set of scores
Given FluModel with id = 1 and 3 GoogleTerm entries
And each GoogleTerm has a score for '2018-01-01'
Then GoogleDate has an entry for FluModel 1 and '2018-01-01'
"""
with self.app.app_context():
for idx in range(3):
flu_model_google_term = FluModelGoogleTerm()
flu_model_google_term.flu_model_id = 1
flu_model_google_term.google_term_id = idx
flu_model_google_term.save()
google_score = GoogleScore(idx, date(2018, 1, 1), 0.1 + idx)
google_score.save()
google_term = GoogleTerm()
google_term.id = idx
google_term.term = 'Term %d' % idx
google_term.save()
set_and_verify_google_dates(1, [date(2018, 1, 1)])
result = DB.session.query(
DB.session.query(GoogleDate)\
.filter(GoogleDate.flu_model_id == 1)\
.filter(GoogleDate.score_date == date(2018, 1, 1))\
.exists()
).scalar()
self.assertTrue(result)
def test_api_returning_zero_values(self):
"""
Scenario: Test run function to attempt calculating model score for a specific date
Given the Google API returns zero for the term temperature
Then the retrieval of the scores and calculation of model scores is skipped
And the requested data is not available on Google API
And a warning log message is printed
"""
with self.app.app_context():
google_date_1 = GoogleDate(1, date.today() - timedelta(days=1))
google_date_1.save()
google_term = GoogleTerm()
google_term.id = 1
google_term.term = 'Term 1'
google_term.save()
flu_model_google_term = FluModelGoogleTerm()
flu_model_google_term.flu_model_id = 1
flu_model_google_term.google_term_id = 1
flu_model_google_term.save()
model_function = ModelFunction()
model_function.flu_model_id = 1
model_function.function_name = 'matlab_function'
model_function.average_window_size = 1
model_function.has_confidence_interval = False
model_function.save()
with patch.multiple(
GoogleApiClient,
fetch_google_scores=DEFAULT,
is_returning_non_zero_for_temperature=DEFAULT
) as patch_dict, patch(
'scheduler.score_calculator.get_dates_missing_model_score'
) as model_score_ctx:
with self.assertLogs(level='WARNING') as logContext:
patch_dict['fetch_google_scores'].return_value = []
patch_dict[
'is_returning_non_zero_for_temperature'].return_value = False
model_score_ctx.return_value = False
score_calculator.run(1, date.today(), date.today())
self.assertListEqual(logContext.output, [
'WARNING:root:Google API has returned zero for the term temperature. Not fetching scores'
])
def test_failed_retrieval_of_google_scores(self):
"""
Scenario: Test run function to attempt calculating model score for a specific date
Given the requested Google date is missing
And the requested data is not available on Google API
Then run function returns without calculating the score
And an error log message is printed
"""
with self.app.app_context():
google_date_1 = GoogleDate(1, date.today() - timedelta(days=1))
google_date_1.save()
google_term = GoogleTerm()
google_term.id = 1
google_term.term = 'Term 1'
google_term.save()
flu_model_google_term = FluModelGoogleTerm()
flu_model_google_term.flu_model_id = 1
flu_model_google_term.google_term_id = 1
flu_model_google_term.save()
model_function = ModelFunction()
model_function.flu_model_id = 1
model_function.function_name = 'matlab_function'
model_function.average_window_size = 1
model_function.has_confidence_interval = False
model_function.save()
with patch.multiple(GoogleApiClient,
fetch_google_scores=DEFAULT,
is_returning_non_zero_for_temperature=DEFAULT
) as patch_dict:
with self.assertLogs(
level='ERROR') as logContext, self.assertRaises(
RuntimeError) as errorCtx:
patch_dict['fetch_google_scores'].return_value = []
patch_dict[
'is_returning_non_zero_for_temperature'].return_value = True
score_calculator.run(1, date.today(), date.today())
self.assertListEqual(
logContext.output,
['ERROR:root:Retrieval of Google scores failed'])
self.assertTrue(
'Retry call to Google API' in str(errorCtx.exception))
def test_get_google_terms_for_model(self):
"""
Scenario: Get list of Google terms for which a particular flu model was created against
Given a FluModelGoogleTerm.flu_model_id value of 1 exists
And GoogleTerms values (1, 'Term 1'), (2, 'Term 2') and (3, 'Term 3') for flu_model_id 1
When model_id = 1
Then the list contains 'Term 1', 'Term 2' and 'Term 3' as tuples
"""
with self.app.app_context():
expected = []
for i in range(3):
flu_model_google_term = FluModelGoogleTerm()
flu_model_google_term.flu_model_id = 1
flu_model_google_term.google_term_id = i
flu_model_google_term.save()
google_term = GoogleTerm()
google_term.id = i
google_term.term = 'Term %d' % i
google_term.save()
expected.append(('Term %d' % i, ))
result = get_google_terms_for_model_id(1)
self.assertListEqual(result, expected)
def test_get_google_terms_and_averages(self):
"""
Scenario: Get the Google terms for a model and their average scores for a date window
"""
with self.app.app_context():
for i in range(3):
flu_model_google_term = FluModelGoogleTerm()
flu_model_google_term.flu_model_id = 1
flu_model_google_term.google_term_id = i
flu_model_google_term.save()
google_term = GoogleTerm()
google_term.id = i
google_term.term = 'Term %d' % i
google_term.save()
google_score = GoogleScore(i, date(2018, 1, 1), 0.5 + i)
google_score.save()
google_score = GoogleScore(i, date(2018, 1, 2), 0.1 + i)
google_score.save()
# Additional FluModel to verify the select works as expected
flu_model_google_term_2 = FluModelGoogleTerm()
flu_model_google_term_2.flu_model_id = 2
flu_model_google_term_2.google_term_id = 3
flu_model_google_term_2.save()
google_term_2 = GoogleTerm()
google_term_2.id = 3
google_term_2.term = 'Term 3'
google_term_2.save()
google_score_2 = GoogleScore(3, date(2018, 1, 1), 1.5)
google_score_2.save()
result = get_google_terms_and_averages(1, 2, date(2018, 1, 2))
expected = [('Term 0', 0.3), ('Term 1', 1.3), ('Term 2', 2.3)]
self.assertListEqual(result, expected)
def test_set_google_date_for_model(self):
"""
Scenario: Persist the date of retrieval of a complete set of scores from Google
Given a flu model with a set of three terms
And a score for each of them for one date
Then the date is persisted on the database
"""
with self.app.app_context():
for i in range(3):
flu_model_google_term = FluModelGoogleTerm()
flu_model_google_term.flu_model_id = 1
flu_model_google_term.google_term_id = i
flu_model_google_term.save()
google_term = GoogleTerm()
google_term.id = i
google_term.term = 'Term %d' % i
google_term.save()
google_score = GoogleScore(i, date(2018, 1, 1), 0.5 + i)
google_score.save()
set_google_date_for_model_id(1, date(2018, 1, 1))
result = DB.session.query(
DB.session.query(GoogleDate).filter_by(
flu_model_id=1, score_date=date(2018, 1,
1)).exists()).scalar()
self.assertTrue(result)
"""
Scenario: Check if ValueError is raised if Google scores are incomplete
"""
flu_model_google_term = FluModelGoogleTerm()
flu_model_google_term.flu_model_id = 1
flu_model_google_term.google_term_id = 3
flu_model_google_term.save()
google_term = GoogleTerm()
google_term.id = 3
google_term.term = 'Term 3'
google_term.save()
with self.assertRaises(ValueError):
set_google_date_for_model_id(1, date(2018, 1, 1))