def validate_data_files() -> bool:
# pylint: disable=too-many-return-statements
# pylint: disable=too-many-branches
"""Check the validity of data files
Note that these checks are computationally intensive, so they probably
should not be included in an automated pipeline. Rather, they are for manual
use.
Data files validated:
* Towers file at :py:const:`mobility_pipeline.data_interface.TOWERS_PATH`
* Voronoi file at :py:const:`mobility_pipeline.data_interface.VORONOI_PATH`
* Mobility file at
:py:const:`mobility_pipeline.data_interface.MOBILITY_PATH`
Returns:
``True`` if all files are valid, ``False`` otherwise.
"""
# Load and validate towers
f = None
try:
f = open(TOWERS_PATH, 'r')
towers_csv = csv.reader(f)
except (FileNotFoundError, IOError) as e:
msg = repr(e)
print(f'INVALID: error opening towers file at {TOWERS_PATH}: {msg}')
return False
else:
print('SUCCESS: Towers file opened')
align_error = validate_tower_index_name_aligned(towers_csv)
if align_error:
print(f'INVALID tower index alignment: {align_error}')
return False
print('SUCCESS: Tower indices aligned')
towers = load_towers(TOWERS_PATH)
finally:
if f:
f.close()
# Load and validate Voronoi
voronoi_errs = validate_voronoi(VORONOI_PATH)
if voronoi_errs:
print(f'INVALID Voronoi: {voronoi_errs}')
return False
print('SUCCESS: Voronoi valid')
voronoi = load_voronoi_cells(VORONOI_PATH)
# Check towers and Voronoi aligned
tower_voronoi_align_err = validate_tower_cells_aligned(voronoi, towers)
if tower_voronoi_align_err:
print(f'INVALID tower-Voronoi alignment: {tower_voronoi_align_err}')
return False
print('SUCCESS: Tower and Voronoi indices aligned')
# Validate mobility data
try:
f = open(MOBILITY_PATH, 'r')
mobility_csv = csv.reader(f)
except (FileNotFoundError, IOError) as e:
msg = repr(e)
print(
f'INVALID: error opening mobility file at {MOBILITY_PATH}: {msg}')
return False
else:
print('SUCCESS: Mobility data loaded')
mobility_err = validate_mobility(list(mobility_csv)[1:])
if mobility_err:
print(f'INVALID mobility data: {mobility_err}')
return False
print('SUCCESS: Mobility data valid')
finally:
if f:
f.close()
# Validate Admins
admin_errs = validate_admins(COUNTRY_ID)
if admin_errs:
print(f'INVALID Admins: {admin_errs}')
return False
print('SUCCESS: Admins valid')
return True
def test_validate_mobility_simple_unordered_3_invalid():
csv = [['20150201', 'br0', 'br0', '13853'],
['20150201', 'br0', 'br0', '13853'],
['20150201', 'br1', 'br3', '13853']]
assert validate_mobility(csv) is not None
def test_validate_mobility_simple_malformed_count_3_invalid():
csv = [['20150201', 'br0', 'br0', '13853'],
['20150201', 'br0', 'br2', '13853a'],
['20150201', 'br1', 'br3', '13853']]
assert validate_mobility(csv) is not None
def test_validate_mobility_simple_negative_count_invalid():
csv = [['20150201', 'br0', 'br0', '13853'],
['20150201', 'br0', 'br2', '-13853'],
['20150201', 'br1', 'br3', '13853']]
assert validate_mobility(csv) is not None
def test_validate_mobility_simple_full_valid():
csv = [['20150201', 'br0', 'br0', '13853'],
['20150201', 'br0', 'br1', '13853'],
['20150201', 'br0', 'br2', '13853']]
assert validate_mobility(csv) is None
