def test_skip_data_preamble(self):
mapreduce_map_input_file = 's3://path/to/fulchesterdata.txt.gz'
os.environ['mapreduce_map_input_file'] = mapreduce_map_input_file
os.environ['map_input_start'] = "0"
mapper = Mapper('Fulchester')
yields = mapper.yields()
self.assertEqual(len(yields), 0)
def test_no_data_available(self):
mapreduce_map_input_file = 's3://path/to/fulchesterdata.txt.gz'
os.environ['mapreduce_map_input_file'] = mapreduce_map_input_file
os.environ['map_input_start'] = "0"
line = '1972 10 --- --- --- ---'
mapper = Mapper(line, True, False)
yields = mapper.yields()
self.assertEqual(len(yields), 0)
def test_station_name(self):
mapreduce_map_input_file = 's3://path/to/fulchesterdata.txt.gz'
os.environ['mapreduce_map_input_file'] = mapreduce_map_input_file
os.environ['map_input_start'] = "0"
line = 'foobar'
mapper = Mapper(line)
station_name = mapper.station_name()
self.assertTrue(station_name == 'fulchester')
def test_provisional_data_can_be_ignored(self):
mapreduce_map_input_file = 's3://path/to/fulchesterdata.txt.gz'
os.environ['mapreduce_map_input_file'] = mapreduce_map_input_file
os.environ['map_input_start'] = "0"
line = '2019 1 7.2* 2.1 10 34.5 69.1 Provisional'
mapper = Mapper(line, True, False)
yields = mapper.yields()
self.assertEqual(len(yields), 0)
def test_tmean(self):
mapreduce_map_input_file = 's3://path/to/fulchesterdata.txt.gz'
os.environ['mapreduce_map_input_file'] = mapreduce_map_input_file
os.environ['map_input_start'] = "0"
line = '2019 1 7.2 2.1 10 34.5 69.1'
mapper = Mapper(line, True, False)
yields = mapper.yields()
self.assertEqual(yields[2]['key'], 'fulchester:tmean:01')
self.assertEqual(yields[2]['value'], 4.65)
def test_tmax_not_available(self):
mapreduce_map_input_file = 's3://path/to/fulchesterdata.txt.gz'
os.environ['mapreduce_map_input_file'] = mapreduce_map_input_file
os.environ['map_input_start'] = "0"
line = '2019 1 --- 2.1 10 34.5 69.1'
mapper = Mapper(line, False, False)
yields = mapper.yields()
for y in yields:
self.assertFalse(y['key'] == 'fulchester:tmax:01')
def test_tmin_no_estimated(self):
mapreduce_map_input_file = 's3://path/to/fulchesterdata.txt.gz'
os.environ['mapreduce_map_input_file'] = mapreduce_map_input_file
os.environ['map_input_start'] = "0"
line = '2019 1 7.2* 2.1* 10 34.5 69.1'
mapper = Mapper(line, False, False)
yields = mapper.yields()
for y in yields:
self.assertFalse(y['key'] == 'fulchester:tmin:01')
def test_skip_nonsensical_data(self):
mapreduce_map_input_file = 's3://path/to/fulchesterdata.txt.gz'
os.environ['mapreduce_map_input_file'] = mapreduce_map_input_file
os.environ['map_input_start'] = "42"
mapper = Mapper('Fulchester')
yields = mapper.yields()
key = mapper.yields()[0]['key']
value = mapper.yields()[0]['value']
self.assertEqual(key, 'mapper_skipped')
comparison = '%s:%d' % (mapreduce_map_input_file, 42)
self.assertEqual(value, comparison)