def test_query_by_issue(self):
"""Query with and without specifying an issue."""
# insert dummy data
def insert_issue(cur, issue, value, record_type):
so_many_nulls = ', '.join(['null'] * 57)
cur.execute(f'''insert into covid_hosp_state_timeseries values (
0, {issue}, 'PA', 20201118, {value}, {so_many_nulls}, '{record_type}'
)''')
with Database.connect() as db:
with db.new_cursor() as cur:
# inserting out of order to test server-side order by
# also inserting two for 20201201 to test tiebreaker.
insert_issue(cur, 20201201, 123, 'T')
insert_issue(cur, 20201201, 321, 'D')
insert_issue(cur, 20201203, 789, 'T')
insert_issue(cur, 20201202, 456, 'T')
# request without issue (defaulting to latest issue)
with self.subTest(name='no issue (latest)'):
response = Epidata.covid_hosp('PA', 20201118)
self.assertEqual(response['result'], 1)
self.assertEqual(len(response['epidata']), 1)
self.assertEqual(response['epidata'][0]['issue'], 20201203)
self.assertEqual(
response['epidata'][0]['critical_staffing_shortage_today_yes'],
789)
# request for specific issue
with self.subTest(name='specific single issue'):
response = Epidata.covid_hosp('PA', 20201118, issues=20201201)
self.assertEqual(response['result'], 1)
self.assertEqual(len(response['epidata']), 1)
self.assertEqual(response['epidata'][0]['issue'], 20201201)
self.assertEqual(
response['epidata'][0]['critical_staffing_shortage_today_yes'],
321)
# request for multiple issues
with self.subTest(name='specific multiple issues'):
issues = Epidata.range(20201201, 20201231)
response = Epidata.covid_hosp('PA', 20201118, issues=issues)
self.assertEqual(response['result'], 1)
self.assertEqual(len(response['epidata']), 3)
rows = response['epidata']
# tiebreaker
self.assertEqual(rows[0]['issue'], 20201201)
self.assertEqual(rows[0]['critical_staffing_shortage_today_yes'],
321)
# server-side order by
self.assertEqual(rows[1]['issue'], 20201202)
self.assertEqual(rows[1]['critical_staffing_shortage_today_yes'],
456)
self.assertEqual(rows[2]['issue'], 20201203)
self.assertEqual(rows[2]['critical_staffing_shortage_today_yes'],
789)
def test_acquire_dataset(self):
"""Acquire a new dataset."""
# make sure the data does not yet exist
with self.subTest(name='no data yet'):
response = Epidata.covid_hosp('MA', Epidata.range(20200101, 20210101))
self.assertEqual(response['result'], -2, response)
# acquire sample data into local database
# mock out network calls to external hosts
with self.subTest(name='first acquisition'), \
patch.object(Network, 'fetch_metadata', return_value=self.test_utils.load_sample_metadata()) as mock_fetch_meta, \
patch.object(Network, 'fetch_dataset', side_effect=[self.test_utils.load_sample_dataset("dataset0.csv"), # dataset for 3/13
self.test_utils.load_sample_dataset("dataset0.csv"), # first dataset for 3/15
self.test_utils.load_sample_dataset()] # second dataset for 3/15
) as mock_fetch:
acquired = Update.run()
self.assertTrue(acquired)
self.assertEqual(mock_fetch_meta.call_count, 1)
# make sure the data now exists
with self.subTest(name='initial data checks'):
response = Epidata.covid_hosp('WY', Epidata.range(20200101, 20210101))
self.assertEqual(response['result'], 1)
self.assertEqual(len(response['epidata']), 1)
row = response['epidata'][0]
self.assertEqual(row['state'], 'WY')
self.assertEqual(row['date'], 20201209)
self.assertEqual(row['issue'], 20210315)
self.assertEqual(row['critical_staffing_shortage_today_yes'], 8)
self.assertEqual(row['total_patients_hospitalized_confirmed_influenza_covid_coverage'], 56)
actual = row['inpatient_bed_covid_utilization']
expected = 0.11729857819905214
self.assertAlmostEqual(actual, expected)
self.assertIsNone(row['critical_staffing_shortage_today_no'])
# expect 61 fields per row (63 database columns, except `id` and `record_type`)
self.assertEqual(len(row), 118)
with self.subTest(name='all date batches acquired'):
response = Epidata.covid_hosp('WY', Epidata.range(20200101, 20210101), issues=20210313)
self.assertEqual(response['result'], 1)
# re-acquisition of the same dataset should be a no-op
with self.subTest(name='second acquisition'), \
patch.object(Network, 'fetch_metadata', return_value=self.test_utils.load_sample_metadata()) as mock_fetch_meta, \
patch.object(Network, 'fetch_dataset', return_value=self.test_utils.load_sample_dataset()) as mock_fetch:
acquired = Update.run()
self.assertFalse(acquired)
# make sure the data still exists
with self.subTest(name='final data checks'):
response = Epidata.covid_hosp('WY', Epidata.range(20200101, 20210101))
self.assertEqual(response['result'], 1)
self.assertEqual(len(response['epidata']), 1)
def test_query_by_issue(self):
"""Query with and without specifying an issue."""
with Database.connect() as db:
with db.new_cursor() as cur:
# inserting out of order to test server-side order by
# also inserting two for 20201201 to test tiebreaker.
self.insert_issue(cur, 20201201, 123, 'T')
self.insert_issue(cur, 20201201, 321, 'D')
self.insert_issue(cur, 20201203, 789, 'T')
self.insert_issue(cur, 20201202, 456, 'T')
# request without issue (defaulting to latest issue)
with self.subTest(name='no issue (latest)'):
response = Epidata.covid_hosp('PA', 20201118)
self.assertEqual(response['result'], 1)
self.assertEqual(len(response['epidata']), 1)
self.assertEqual(response['epidata'][0]['issue'], 20201203)
self.assertEqual(
response['epidata'][0]['critical_staffing_shortage_today_yes'],
789)
# request for specific issue
with self.subTest(name='specific single issue'):
response = Epidata.covid_hosp('PA', 20201118, issues=20201201)
self.assertEqual(response['result'], 1)
self.assertEqual(len(response['epidata']), 1)
self.assertEqual(response['epidata'][0]['issue'], 20201201)
self.assertEqual(
response['epidata'][0]['critical_staffing_shortage_today_yes'],
321)
# request for multiple issues
with self.subTest(name='specific multiple issues'):
issues = Epidata.range(20201201, 20201231)
response = Epidata.covid_hosp('PA', 20201118, issues=issues)
self.assertEqual(response['result'], 1)
self.assertEqual(len(response['epidata']), 3)
rows = response['epidata']
# tiebreaker
self.assertEqual(rows[0]['issue'], 20201201)
self.assertEqual(rows[0]['critical_staffing_shortage_today_yes'],
321)
# server-side order by
self.assertEqual(rows[1]['issue'], 20201202)
self.assertEqual(rows[1]['critical_staffing_shortage_today_yes'],
456)
self.assertEqual(rows[2]['issue'], 20201203)
self.assertEqual(rows[2]['critical_staffing_shortage_today_yes'],
789)
def test_query_by_issue(self):
"""Query with and without specifying an issue."""
# insert dummy data
def insert_issue(cur, issue, value):
so_many_nulls = ', '.join(['null'] * 51)
cur.execute(f'''insert into covid_hosp values (
0, {issue}, 'PA', 20201118, {value}, {so_many_nulls}
)''')
with Database.connect() as db:
with db.new_cursor() as cur:
# inserting out of order to test server-side order by
insert_issue(cur, 20201201, 123)
insert_issue(cur, 20201203, 789)
insert_issue(cur, 20201202, 456)
# request without issue (defaulting to latest issue)
with self.subTest(name='no issue (latest)'):
response = Epidata.covid_hosp('PA', 20201118)
self.assertEqual(response['result'], 1)
self.assertEqual(len(response['epidata']), 1)
self.assertEqual(response['epidata'][0]['issue'], 20201203)
self.assertEqual(response['epidata'][0]['hospital_onset_covid'],
789)
# request for specific issue
with self.subTest(name='specific single issue'):
response = Epidata.covid_hosp('PA', 20201118, issues=20201201)
self.assertEqual(response['result'], 1)
self.assertEqual(len(response['epidata']), 1)
self.assertEqual(response['epidata'][0]['issue'], 20201201)
self.assertEqual(response['epidata'][0]['hospital_onset_covid'],
123)
# request for multiple issues
with self.subTest(name='specific multiple issues'):
issues = Epidata.range(20201201, 20201231)
response = Epidata.covid_hosp('PA', 20201118, issues=issues)
self.assertEqual(response['result'], 1)
self.assertEqual(len(response['epidata']), 3)
rows = response['epidata']
self.assertEqual(rows[0]['issue'], 20201201)
self.assertEqual(rows[0]['hospital_onset_covid'], 123)
self.assertEqual(rows[1]['issue'], 20201202)
self.assertEqual(rows[1]['hospital_onset_covid'], 456)
self.assertEqual(rows[2]['issue'], 20201203)
self.assertEqual(rows[2]['hospital_onset_covid'], 789)
def test_acquire_dataset(self):
"""Acquire a new dataset."""
# only mock out network calls to external hosts
mock_network = MagicMock()
mock_network.fetch_metadata.return_value = \
self.test_utils.load_sample_metadata()
mock_network.fetch_dataset.return_value = \
self.test_utils.load_sample_dataset()
# make sure the data does not yet exist
with self.subTest(name='no data yet'):
response = Epidata.covid_hosp('MA',
Epidata.range(20200101, 20210101))
self.assertEqual(response['result'], -2)
# acquire sample data into local database
with self.subTest(name='first acquisition'):
acquired = Update.run(network=mock_network)
self.assertTrue(acquired)
# make sure the data now exists
with self.subTest(name='initial data checks'):
response = Epidata.covid_hosp('WY',
Epidata.range(20200101, 20210101))
self.assertEqual(response['result'], 1)
self.assertEqual(len(response['epidata']), 1)
row = response['epidata'][0]
self.assertEqual(row['state'], 'WY')
self.assertEqual(row['date'], 20200826)
self.assertEqual(row['issue'], 20201116)
self.assertEqual(row['critical_staffing_shortage_today_yes'], 2)
actual = row['inpatient_bed_covid_utilization']
expected = 0.011946591707659873
self.assertAlmostEqual(actual, expected)
self.assertIsNone(row['critical_staffing_shortage_today_no'])
# expect 61 fields per row (63 database columns, except `id` and `record_type`)
self.assertEqual(len(row), 61)
# re-acquisition of the same dataset should be a no-op
with self.subTest(name='second acquisition'):
acquired = Update.run(network=mock_network)
self.assertFalse(acquired)
# make sure the data still exists
with self.subTest(name='final data checks'):
response = Epidata.covid_hosp('WY',
Epidata.range(20200101, 20210101))
self.assertEqual(response['result'], 1)
self.assertEqual(len(response['epidata']), 1)
def test_acquire_dataset(self):
"""Acquire a new dataset."""
# only mock out network calls to external hosts
mock_network = MagicMock()
mock_network.fetch_metadata.return_value = \
self.test_utils.load_sample_metadata()
mock_network.fetch_dataset.return_value = \
self.test_utils.load_sample_dataset()
# make sure the data does not yet exist
with self.subTest(name='no data yet'):
response = Epidata.covid_hosp('MA',
Epidata.range(20200101, 20210101))
self.assertEqual(response['result'], -2)
# acquire sample data into local database
with self.subTest(name='first acquisition'):
acquired = Update.run(network_impl=mock_network)
self.assertTrue(acquired)
# make sure the data now exists
with self.subTest(name='initial data checks'):
response = Epidata.covid_hosp('MA',
Epidata.range(20200101, 20210101))
self.assertEqual(response['result'], 1)
self.assertEqual(len(response['epidata']), 1)
row = response['epidata'][0]
self.assertEqual(row['state'], 'MA')
self.assertEqual(row['date'], 20200510)
self.assertEqual(row['issue'], 20201116)
self.assertEqual(row['hospital_onset_covid'], 53)
actual = row['inpatient_bed_covid_utilization']
expected = 0.21056656682174496
self.assertAlmostEqual(actual, expected)
self.assertIsNone(row['adult_icu_bed_utilization'])
# expect 55 fields per row (56 database columns, except `id`)
self.assertEqual(len(row), 55)
# re-acquisition of the same dataset should be a no-op
with self.subTest(name='second acquisition'):
acquired = Update.run(network_impl=mock_network)
self.assertFalse(acquired)
# make sure the data still exists
with self.subTest(name='final data checks'):
response = Epidata.covid_hosp('MA',
Epidata.range(20200101, 20210101))
self.assertEqual(response['result'], 1)
self.assertEqual(len(response['epidata']), 1)
def test_acquire_specific_issue(self):
"""Acquire a new dataset."""
# make sure the data does not yet exist
with self.subTest(name='no data yet'):
response = Epidata.covid_hosp('MA', Epidata.range(20200101, 20210101))
self.assertEqual(response['result'], -2)
# acquire sample data into local database
# mock out network calls to external hosts
with Database.connect() as db:
pre_max_issue = db.get_max_issue()
self.assertEqual(pre_max_issue, pd.Timestamp('1900-01-01 00:00:00'))
with self.subTest(name='first acquisition'), \
patch.object(Network, 'fetch_metadata', return_value=self.test_utils.load_sample_metadata()) as mock_fetch_meta, \
patch.object(Network, 'fetch_dataset', side_effect=[self.test_utils.load_sample_dataset("dataset0.csv")]
) as mock_fetch:
acquired = Utils.update_dataset(Database,
Network,
date(2021, 3, 12),
date(2021, 3, 14))
with Database.connect() as db:
post_max_issue = db.get_max_issue()
self.assertEqual(post_max_issue, pd.Timestamp('2021-03-13 00:00:00'))
self.assertTrue(acquired)
def test_query_by_as_of(self):
with Database.connect() as db:
with db.new_cursor() as cur:
self.insert_issue(cur, 20201101, 0, 'T')
self.insert_issue(cur, 20201102, 1, 'D')
self.insert_issue(cur, 20201103, 2, 'D')
self.insert_issue(cur, 20201103, 3, 'T')
self.insert_issue(cur, 20201104, 4, 'T')
with self.subTest(name='as_of with multiple issues'):
response = Epidata.covid_hosp('PA', 20201118, as_of=20201103)
self.assertEqual(response['result'], 1)
self.assertEqual(len(response['epidata']), 1)
self.assertEqual(response['epidata'][0]['issue'], 20201103)
self.assertEqual(
response['epidata'][0]['critical_staffing_shortage_today_yes'],
2)
