def test_sequentialInvalidData_returnsProcessedAndCorrectedData(self):
lines = """3838 20170321 1400 20170321 0900 2.422 -84.75 38.09 10.0 9.5 10.0 9.2 0.0 68 0 139 0 30 0 C 9.5 0 10.4 0 8.9 0 89 0 0.300 0.303 0.323 0.312 0.409 7.4 7.5 7.5 7.6 8.4
63838 20170321 1500 20170321 1000 2.422 -84.75 38.09 10.6 10.3 10.6 10.0 0.0 118 0 162 0 80 0 C 10.8 0 11.3 0 10.4 0 84 0 0.301 0.305 0.322 0.311 0.408 7.6 7.7 7.5 7.5 8.4
63838 20170321 1600 20170321 1100 -9.000 -84.75 38.09 -9999.0 -9999.0 -9999.0 -9999.0 -9999.0 -99999 0 -99999 0 -99999 0 U -9999.0 0 -9999.0 0 -9999.0 0 -9999 0 -99.000 -99.000 -99.000 -99.000 -99.000 -9999.0 -9999.0 -9999.0 -9999.0 -9999.0
63838 20170321 1700 20170321 1200 2.422 -84.75 38.09 -9999.0 -9999.0 -9999.0 -9999.0 -9999.0 0 3 0 3 0 3 C 12.4 3 18.0 3 0.0 3 -9999 0 -99.000 -99.000 -99.000 -99.000 -99.000 -9999.0 -9999.0 -9999.0 -9999.0 -9999.0
63838 20170321 1800 20170321 1300 2.422 -84.75 38.09 -9999.0 -9999.0 -9999.0 -9999.0 -9999.0 0 3 0 3 0 3 C 17.6 3 19.1 3 16.5 3 -9999 0 -99.000 -99.000 -99.000 -99.000 -99.000 -9999.0 -9999.0 -9999.0 -9999.0 -9999.0
63838 20170321 1900 20170321 1400 2.422 -84.75 38.09 -9999.0 -9999.0 -9999.0 -9999.0 -9999.0 685 3 1105 3 302 3 C 19.7 3 26.5 3 0.0 3 -9999 0 -99.000 -99.000 -99.000 -99.000 -99.000 -9999.0 -9999.0 -9999.0 -9999.0 -9999.0
63838 20170321 2000 20170321 1500 2.422 -84.75 38.09 14.6 14.4 14.9 13.8 0.0 608 0 770 0 357 0 C 22.3 0 24.2 0 20.0 0 62 0 0.296 0.302 0.322 0.311 0.408 9.8 9.6 7.9 7.6 8.4
63838 20170321 2100 20170321 1600 2.422 -84.75 38.09 15.3 14.8 15.3 14.5 0.0 405 0 610 0 291 0 C 19.7 0 21.8 0 17.7 0 58 0 0.295 0.302 0.320 0.311 0.408 10.3 10.1 8.1 7.6 8.4"""
rows = [re.split(r"\s+", line) for line in lines.splitlines()]
first_valid_row = rows[0]
last_valid_row = rows[7]
temperature = USCRNDataType(8, -9999.0)
humidity = USCRNDataType(26, -9999, 27)
solar_global = USCRNDataType(13, -99999, 14)
data_types = [temperature, humidity, solar_global]
assert aGosedWeather._extract_data(first_valid_row, last_valid_row,
rows,
data_types) == [[10.0, 89, 68],
[10.6, 84, 118],
[11.4, 80, 216],
[12.2, 75, 314],
[13.0, 71, 412],
[13.8, 66, 510],
[14.6, 62, 608],
[15.3, 58, 405]]
def test_nonsequentialInvalidData_returnsProcessedAndCorrectedData(
self):
lines = """63838 20170320 2100 20170320 1600 2.422 -84.75 38.09 17.3 16.2 17.3 14.9 0.0 488 0 584 0 166 0 C 17.8 0 19.4 0 16.2 0 46 0 0.302 0.304 0.322 0.309 0.403 7.5 7.3 6.7 7.3 8.4
63838 20170320 2200 20170320 1700 2.422 -84.75 38.09 -9999.0 17.9 18.3 17.4 0.0 388 0 519 0 195 0 C 18.7 0 19.4 0 18.2 0 44 0 0.303 0.305 0.323 0.308 0.403 7.9 7.8 6.8 7.3 8.4
63838 20170320 2300 20170320 1800 2.422 -84.75 38.09 -9999.0 17.9 18.4 17.6 0.0 100 0 296 0 39 0 C 16.2 0 18.5 0 15.2 0 47 0 0.299 0.303 0.320 0.312 0.413 8.2 8.1 7.0 7.3 8.4
63838 20170321 0000 20170320 1900 2.422 -84.75 38.09 13.9 15.4 18.0 13.6 0.0 7 0 66 0 0 0 C 11.2 0 15.6 0 9.2 0 57 0 0.301 0.305 0.323 0.310 0.403 8.3 8.3 7.2 7.3 8.4
63838 20170321 0100 20170320 2000 2.422 -84.75 38.09 13.1 13.5 14.0 13.1 0.0 0 0 0 0 0 0 C 9.0 0 9.2 0 8.8 0 -9999 0 0.299 0.305 0.321 0.309 0.408 8.2 8.3 7.3 7.3 8.4
63838 20170321 0200 20170320 2100 2.422 -84.75 38.09 11.6 12.5 13.3 11.6 0.0 -99999 0 0 0 0 0 C 8.5 0 8.8 0 8.1 0 63 0 0.301 0.304 0.323 0.308 0.399 8.1 8.2 7.4 7.3 8.4
63838 20170321 0300 20170320 2200 2.422 -84.75 38.09 -9999.0 11.7 11.9 11.5 0.0 -99999 0 0 0 0 0 C 9.4 0 10.2 0 8.4 0 73 0 0.300 0.301 0.324 0.312 0.401 8.1 8.1 7.4 7.4 8.4
63838 20170321 0400 20170320 2300 2.422 -84.75 38.09 11.4 11.7 11.9 11.4 0.0 -99999 0 0 0 0 0 C 10.0 0 10.2 0 8.0 0 73 0 0.301 0.303 0.323 0.309 0.408 7.9 8.1 7.5 7.4 8.4
63838 20170321 0500 20170321 0000 2.422 -84.75 38.09 10.3 10.6 11.4 9.9 0.0 -99999 0 0 0 0 0 C 7.4 0 8.3 0 6.2 0 76 0 0.301 0.303 0.324 0.311 0.404 7.9 8.0 7.6 7.4 8.4
63838 20170321 0600 20170321 0100 2.422 -84.75 38.09 9.6 9.9 10.9 9.2 0.0 0 0 0 0 0 0 C 6.3 0 6.8 0 5.7 0 -9999 0 0.300 0.305 0.322 0.308 0.407 7.7 8.0 7.6 7.4 8.4
63838 20170321 0700 20170321 0200 2.422 -84.75 38.09 9.0 9.5 10.0 9.0 0.0 0 0 0 0 0 0 C 5.9 0 6.6 0 5.3 0 79 0 0.301 0.303 0.322 0.310 0.406 7.7 7.8 7.6 7.5 8.4
63838 20170321 0800 20170321 0300 2.422 -84.75 38.09 9.1 8.9 9.3 8.3 0.0 0 0 0 0 0 0 C 6.4 0 7.3 0 5.2 0 83 0 0.302 0.304 0.323 0.308 0.404 7.6 7.6 7.6 7.4 8.4"""
rows = [re.split(r"\s+", line) for line in lines.splitlines()]
first_valid_row = rows[0]
last_valid_row = rows[len(rows) - 1]
temperature = USCRNDataType(8, -9999.0)
humidity = USCRNDataType(26, -9999, 27)
solar_global = USCRNDataType(13, -99999, 14)
data_types = [temperature, solar_global, humidity]
assert aGosedWeather._extract_data(first_valid_row, last_valid_row,
rows,
data_types) == [[17.3, 488, 46],
[16.2, 388, 44],
[15.0, 100, 47],
[13.9, 7, 57],
[13.1, 0, 60],
[11.6, 0, 63],
[11.5, 0, 73],
[11.4, 0, 73],
[10.3, 0, 76],
[9.6, 0, 78],
[9.0, 0, 79],
[9.1, 0, 83]]
def test_allValidData_returnsProcessedData(self):
lines = """63838 20170914 1400 20170914 0900 2.422 -84.75 38.09 16.9 16.4 16.9 16.2 0.0 116 0 198 0 80 0 C 16.6 0 17.3 0 16.2 0 94 0 0.382 0.363 0.333 0.308 0.402 18.7 19.0 19.1 19.5 19.7
63838 20170914 1500 20170914 1000 2.422 -84.75 38.09 18.2 17.6 18.3 16.8 0.0 312 0 738 0 169 0 C 18.2 0 19.7 0 17.2 0 89 0 0.380 0.363 0.338 0.307 0.392 18.8 19.0 19.1 19.6 19.7
63838 20170914 1600 20170914 1100 2.422 -84.75 38.09 18.9 18.7 19.2 18.2 0.0 382 0 1092 0 245 0 C 19.9 0 21.0 0 18.9 0 83 0 0.382 0.364 0.333 0.309 0.403 19.1 19.3 19.1 19.5 19.7
63838 20170914 1700 20170914 1200 2.422 -84.75 38.09 20.2 19.8 20.6 19.0 0.0 461 0 1289 0 296 0 C 20.9 0 22.6 0 20.1 0 79 0 0.383 0.363 0.338 0.310 0.394 19.5 19.5 19.2 19.5 19.6
63838 20170914 1800 20170914 1300 2.422 -84.75 38.09 20.3 20.5 21.3 20.2 0.0 471 0 1228 0 213 0 C 21.8 0 24.2 0 20.2 0 75 0 0.384 0.361 0.338 0.309 0.395 19.8 19.9 19.2 19.5 19.6
63838 20170914 1900 20170914 1400 2.422 -84.75 38.09 20.5 20.3 20.6 20.1 0.0 218 0 310 0 152 0 C 20.3 0 21.0 0 19.8 0 76 0 0.378 0.362 0.336 0.308 0.402 20.0 20.3 19.3 19.5 19.6
63838 20170914 2000 20170914 1500 2.422 -84.75 38.09 20.4 20.5 20.6 20.3 0.0 207 0 249 0 149 0 C 20.5 0 20.8 0 20.2 0 76 0 0.382 0.363 0.337 0.306 0.401 20.2 20.3 19.4 19.5 19.8
63838 20170914 2100 20170914 1600 2.422 -84.75 38.09 21.0 20.8 21.2 20.3 0.0 271 0 493 0 152 0 C 21.1 0 21.9 0 19.9 0 74 0 0.377 0.360 0.338 0.308 0.402 20.4 20.5 19.6 19.4 19.6
63838 20170914 2200 20170914 1700 2.422 -84.75 38.09 21.1 21.1 21.4 21.0 0.0 213 0 327 0 130 0 C 21.5 0 22.1 0 21.0 0 71 0 0.375 0.360 0.338 0.307 0.398 20.5 20.6 19.6 19.5 19.6
63838 20170914 2300 20170914 1800 2.422 -84.75 38.09 20.5 20.9 21.1 20.5 0.0 116 0 213 0 43 0 C 21.0 0 21.9 0 19.7 0 73 0 0.379 0.358 0.339 0.307 0.400 20.6 20.7 19.8 19.5 19.7
63838 20170915 0000 20170914 1900 2.422 -84.75 38.09 19.7 20.3 20.7 19.7 0.0 38 0 116 0 0 0 C 19.0 0 20.8 0 16.9 0 77 0 0.378 0.355 0.337 0.310 0.404 20.6 20.7 19.9 19.5 19.7"""
rows = [re.split(r"\s+", line) for line in lines.splitlines()]
first_valid_row = rows[0]
last_valid_row = rows[len(rows) - 1]
temperature = USCRNDataType(8, -9999.0)
humidity = USCRNDataType(26, -9999, 27)
solar_global = USCRNDataType(13, -99999, 14)
data_types = [temperature, humidity, solar_global]
assert aGosedWeather._extract_data(first_valid_row, last_valid_row,
rows,
data_types) == [[16.9, 94, 116],
[18.2, 89, 312],
[18.9, 83, 382],
[20.2, 79, 461],
[20.3, 75, 471],
[20.5, 76, 218],
[20.4, 76, 207],
[21.0, 74, 271],
[21.1, 71, 213],
[20.5, 73, 116],
[19.7, 77, 38]]
def test_invalidData_returnsAveragedAndCorrectedProcessedData(self):
lines = """63838 20170105 0135 20170104 2035 2 -84.75 38.09 -5.4 0.0 0 0 -6.4 C 0 67 0 0.351 5.7 1192 0 2.06 0
63838 20170105 0140 20170104 2040 2 -84.75 38.09 -5.3 0.0 0 0 -6.4 C 0 67 0 0.351 5.6 1194 0 1.99 0
63838 20170105 0145 20170104 2045 2 -84.75 38.09 -5.6 0.0 0 0 -6.5 C 0 68 0 0.351 5.6 1193 0 2.07 0
63838 20170105 0150 20170104 2050 2 -84.75 38.09 -5.7 0.0 0 0 -6.6 C 0 69 0 0.354 5.6 1192 0 1.81 0
63838 20170105 0155 20170104 2055 2 -84.75 38.09 -5.8 0.0 0 0 -6.7 C 0 69 0 0.353 5.6 1194 0 1.77 0
63838 20170105 0200 20170104 2100 2 -84.75 38.09 -5.7 0.0 0 0 -6.7 C 0 69 0 0.352 5.6 1193 0 -99.00 0
63838 20170105 0205 20170104 2105 2 -84.75 38.09 -5.6 0.0 0 0 -6.6 C 0 68 0 0.352 5.5 1194 0 -99.00 0
63838 20170105 0210 20170104 2110 2 -84.75 38.09 -5.6 0.0 0 0 -6.6 C 0 68 0 0.351 5.5 1193 0 -99.00 0
63838 20170105 0215 20170104 2115 2 -84.75 38.09 -5.6 0.0 0 0 -6.3 C 0 68 0 0.351 5.5 1194 0 -99.00 0
63838 20170105 0220 20170104 2120 2 -84.75 38.09 -5.4 0.0 0 0 -5.7 C 0 69 0 0.353 5.5 1194 0 -99.00 0
63838 20170105 0225 20170104 2125 2 -84.75 38.09 -5.1 0.0 0 0 -5.4 C 0 68 0 0.352 5.5 1192 0 -99.00 0
63838 20170105 0230 20170104 2130 2 -84.75 38.09 -4.8 0.0 0 0 -5.1 C 0 66 0 0.353 5.4 1193 0 -99.00 0
63838 20170105 0235 20170104 2135 2 -84.75 38.09 -4.7 0.0 0 0 -4.7 C 0 65 0 0.353 5.4 1190 0 -99.00 0
63838 20170105 0240 20170104 2140 2 -84.75 38.09 -4.7 0.0 0 0 -4.5 C 0 66 0 0.350 5.4 1193 0 -99.00 0
63838 20170105 0245 20170104 2145 2 -84.75 38.09 -4.6 0.0 0 0 -4.4 C 0 66 0 0.353 5.4 1193 0 2.21 3
63838 20170105 0250 20170104 2150 2 -84.75 38.09 -4.6 0.0 0 0 -4.3 C 0 65 0 0.352 5.3 1193 0 2.51 3
63838 20170105 0255 20170104 2155 2 -84.75 38.09 -4.6 0.0 0 0 -4.3 C 0 64 0 0.351 5.3 1194 0 2.23 3
63838 20170105 0300 20170104 2200 2 -84.75 38.09 -4.6 0.0 0 0 -4.3 C 0 65 0 0.351 5.3 1193 0 1.89 0
63838 20170105 0305 20170104 2205 2 -84.75 38.09 -4.5 0.0 0 0 -4.3 C 0 65 0 0.352 5.3 1194 0 2.30 0
63838 20170105 0310 20170104 2210 2 -84.75 38.09 -4.6 0.0 0 0 -4.2 C 0 66 0 0.351 5.3 1191 0 1.86 0
63838 20170105 0315 20170104 2215 2 -84.75 38.09 -4.6 0.0 0 0 -4.2 C 0 67 0 0.353 5.3 1195 0 1.98 0
63838 20170105 0320 20170104 2220 2 -84.75 38.09 -4.8 0.0 0 0 -4.2 C 0 67 0 0.350 5.2 1195 0 2.29 0
63838 20170105 0325 20170104 2225 2 -84.75 38.09 -4.7 0.0 0 0 -4.3 C 0 67 0 0.351 5.2 1193 0 2.53 0
63838 20170105 0330 20170104 2230 2 -84.75 38.09 -4.7 0.0 0 0 -4.3 C 0 66 0 0.351 5.2 1195 0 2.61 0"""
rows = [re.split(r"\s+", line) for line in lines.splitlines()]
first_valid_row = rows[0]
last_valid_row = rows[len(rows) - 1]
wind_speed = USCRNDataType(21, -99.00, 22, lambda x: round(x, 2))
data_types = [wind_speed]
assert aGosedWeather._extract_data(
first_valid_row,
last_valid_row,
rows,
data_types,
is_subhourly_data=True) == [[1.86], [2.06]]