def test_load_mapping(self):
"""
Test if mapping data is loaded properly
"""
with my_vcr.use_cassette("foia-gov-years.yaml"):
response = requests.get(processing_time_scraper.YEARS_URL)
html = response.text
years = processing_time_scraper.get_years(html)
mapping = processing_time_scraper.load_mapping(years)
# Check simple mapping (spelling error on foia.gov data)
foia_data_key = 'bureau of alcohal, tobacco, ' + \
'firearms and explosives_doj_2013'
self.assertEqual(
mapping[foia_data_key][0],
'bureau of alcohol, tobacco, firearms, and explosives_doj_2013')
# Check complex mapping instance (multiple names on foia.gov data)
self.assertEqual(
mapping['office of information and technology (005)_va_2013'],
mapping['office of information and technology_va_2013'])
# Check complex mapping instance (multiple names on foia_hub)
yaml_key_1 = "surface transportation board_dot_2013"
yaml_key_2 = "surface transportation board_stb_2013"
foia_data_key = "surface transportation board_stb_2013"
self.assertEqual(
[yaml_key_1, yaml_key_2],
sorted(mapping[foia_data_key]))
def test_apply_mapping(self):
"""
Verify that foia.gov/data keys are changed to keys compatiable with
yaml keys
"""
with my_vcr.use_cassette("foia-gov-years.yaml"):
response = requests.get(processing_time_scraper.YEARS_URL)
html = response.text
years = processing_time_scraper.get_years(html)
mapping = processing_time_scraper.load_mapping(years)
# Check if items without mapping pass through without change
foia_data_key = 'non_mapped_office_2013'
test_data = {foia_data_key: {'simple_median_days': 3}}
mapped_test_data = processing_time_scraper.apply_mapping(
test_data, mapping)
self.assertEqual(mapped_test_data[foia_data_key],
test_data[foia_data_key])
# Test simple transformations (spelling error on foia.gov data)
foia_data_key = 'bureau of alcohal, tobacco, ' + \
'firearms and explosives_doj_2013'
yaml_key = 'bureau of alcohol, tobacco, firearms,' + \
' and explosives_doj_2013'
test_data = {foia_data_key: {'simple_median_days': 3}}
mapped_test_data = processing_time_scraper.apply_mapping(
test_data, mapping)
self.assertEqual(mapped_test_data[yaml_key], test_data[foia_data_key])
# Test complex transformations (multiple names on foia.gov data)
foia_data_key_1 = 'office of information and technology (005)_va_2013'
foia_data_key_2 = 'office of information and technology_va_2008'
yaml_key_1 = 'office of assistant secretary for ' + \
'information and technology_va_2013'
yaml_key_2 = 'office of assistant secretary for ' + \
'information and technology_va_2008'
test_data_1 = {foia_data_key_1: {'simple_median_days': 10}}
test_data_2 = {foia_data_key_2: {'simple_median_days': 50}}
mapped_test_data_1 = processing_time_scraper.apply_mapping(
test_data_1, mapping)
mapped_test_data_2 = processing_time_scraper.apply_mapping(
test_data_2, mapping)
self.assertEqual(mapped_test_data_1[yaml_key_1],
test_data_1[foia_data_key_1])
self.assertEqual(mapped_test_data_2[yaml_key_2],
test_data_2[foia_data_key_2])
# Test complex transformations (multiple names on foia.gov data)
foia_data_key = 'surface transportation board_stb_2013'
yaml_key_1 = 'surface transportation board_dot_2013'
yaml_key_2 = 'surface transportation board_stb_2013'
test_data = {foia_data_key: {'simple_median_days': 10}}
mapped_test_data = processing_time_scraper.apply_mapping(
test_data, mapping)
self.assertEqual(mapped_test_data[yaml_key_1],
mapped_test_data[yaml_key_2])
def test_load_mapping(self):
"""
Test if mapping data is loaded properly
"""
with my_vcr.use_cassette("foia-gov-years.yaml"):
response = requests.get(processing_time_scraper.YEARS_URL)
html = response.text
years = processing_time_scraper.get_years(html)
mapping = processing_time_scraper.load_mapping(years)
# Check simple mapping (spelling error on foia.gov data)
foia_data_key = 'bureau of alcohal, tobacco, ' + \
'firearms and explosives_doj_2013'
self.assertEqual(
mapping[foia_data_key][0],
'bureau of alcohol, tobacco, firearms, and explosives_doj_2013')
# Check complex mapping instance (multiple names on foia.gov data)
self.assertEqual(
mapping['office of information and technology (005)_va_2013'],
mapping['office of information and technology_va_2013'])
# Check complex mapping instance (multiple names on foia_hub)
yaml_key_1 = "surface transportation board_dot_2013"
yaml_key_2 = "surface transportation board_stb_2013"
foia_data_key = "surface transportation board_stb_2013"
self.assertEqual(
[yaml_key_1, yaml_key_2],
sorted(mapping[foia_data_key]))
def test_get_years(self):
""" Verify that the correct years are retrieved """
with my_vcr.use_cassette("foia-gov-years.yaml"):
response = requests.get(processing_time_scraper.YEARS_URL)
html = response.text
years = processing_time_scraper.get_years(html)
years = sorted(years)
self.assertEqual(['2008', '2009', '2010', '2011'], years[0:4])
def test_get_years(self):
""" Verify that the correct years are retrieved """
with my_vcr.use_cassette("foia-gov-years.yaml"):
response = requests.get(processing_time_scraper.YEARS_URL)
html = response.text
years = processing_time_scraper.get_years(html)
years = sorted(years)
self.assertEqual(['2008', '2009', '2010', '2011'], years[0:4])
def test_apply_mapping(self):
"""
Verify that foia.gov/data keys are changed to keys compatiable with
yaml keys
"""
with my_vcr.use_cassette("foia-gov-years.yaml"):
response = requests.get(processing_time_scraper.YEARS_URL)
html = response.text
years = processing_time_scraper.get_years(html)
mapping = processing_time_scraper.load_mapping(years)
# Check if items without mapping pass through without change
foia_data_key = 'non_mapped_office_2013'
test_data = {foia_data_key: {'simple_median_days': 3}}
mapped_test_data = processing_time_scraper.apply_mapping(
test_data, mapping)
self.assertEqual(
mapped_test_data[foia_data_key], test_data[foia_data_key])
# Test simple transformations (spelling error on foia.gov data)
foia_data_key = 'bureau of alcohal, tobacco, ' + \
'firearms and explosives_doj_2013'
yaml_key = 'bureau of alcohol, tobacco, firearms,' + \
' and explosives_doj_2013'
test_data = {foia_data_key: {'simple_median_days': 3}}
mapped_test_data = processing_time_scraper.apply_mapping(
test_data, mapping)
self.assertEqual(
mapped_test_data[yaml_key], test_data[foia_data_key])
# Test complex transformations (multiple names on foia.gov data)
foia_data_key_1 = 'office of information and technology (005)_va_2013'
foia_data_key_2 = 'office of information and technology_va_2008'
yaml_key_1 = 'office of assistant secretary for ' + \
'information and technology_va_2013'
yaml_key_2 = 'office of assistant secretary for ' + \
'information and technology_va_2008'
test_data_1 = {foia_data_key_1: {'simple_median_days': 10}}
test_data_2 = {foia_data_key_2: {'simple_median_days': 50}}
mapped_test_data_1 = processing_time_scraper.apply_mapping(
test_data_1, mapping)
mapped_test_data_2 = processing_time_scraper.apply_mapping(
test_data_2, mapping)
self.assertEqual(
mapped_test_data_1[yaml_key_1], test_data_1[foia_data_key_1])
self.assertEqual(
mapped_test_data_2[yaml_key_2], test_data_2[foia_data_key_2])
# Test complex transformations (multiple names on foia.gov data)
foia_data_key = 'surface transportation board_stb_2013'
yaml_key_1 = 'surface transportation board_dot_2013'
yaml_key_2 = 'surface transportation board_stb_2013'
test_data = {foia_data_key: {'simple_median_days': 10}}
mapped_test_data = processing_time_scraper.apply_mapping(
test_data, mapping)
self.assertEqual(
mapped_test_data[yaml_key_1],
mapped_test_data[yaml_key_2])