def list_as_json(self) -> str:
"""Return list of DatamapLine objects parsed from filepath as json."""
try:
lst_of_objs = datamap_reader(self.datamap_path)
except DatamapNotCSVException:
raise
return json.dumps(lst_of_objs, cls=DatamapEncoder)
def check_aux_files(config: Config):
"""Check the presence of the blank template, the datamap and the validity of the datmap"""
blank_t = check_for_blank(config)
if blank_t[0]:
logger.info(
f"Blank template named {blank_t[1]} present in input directory.")
else:
logger.critical(
f"No blank template present. Config requires a file called "
f"{config.config_parser['DEFAULT']['blank file name']} in input directory."
)
sys.exit(0)
dm_t = check_for_datamap(config)
if dm_t[0]:
logger.info(
f"Datamap file named {dm_t[1]} present in input directory.")
else:
logger.critical(
f"No datamap file present. Config requires a file called "
f"{config.config_parser['DEFAULT']['datamap file name']} in input directory."
)
sys.exit(0)
# if we get this far, there is a datamap file, so we can run this
dm_name = config.config_parser["DEFAULT"]["datamap file name"]
if datamap_reader(config.PLATFORM_DOCS_DIR / "input" / dm_name):
logger.info(
"Datamap file passes tests. Check any WARNING messages. Ok to proceed."
)
else:
logger.critical("Datamap tests failed")
def test_datamap_with_two_headers(datamap_two_headers):
with pytest.raises(MalFormedCSVHeaderException) as excinfo:
data = datamap_reader(datamap_two_headers) # noqa
msg = excinfo.value.args[0]
assert (
msg ==
"Datamap contains only two headers - need at least three to proceed. Quitting."
)
def test_incorrect_headers_are_coerced_or_flagged(
datamap_moderately_bad_headers):
data = datamap_reader(datamap_moderately_bad_headers)
# using same test as above because even though this datamap file has bad keys, we
# accept them because they are not too bad...
assert data[14].key == "Bad Spacing"
assert data[14].sheet == "Introduction"
assert data[14].cellref == "C35"
assert data[14].data_type == "TEXT"
def test_datamap_type_is_optional(datamap_no_type_col):
"""We want the datamap to be processed even if it doesn't have a type col, which
should be optional.
"""
data = datamap_reader(datamap_no_type_col)
assert data[14].key == "Bad Spacing"
assert data[14].sheet == "Introduction"
assert data[14].cellref == "C35"
assert data[14].data_type is None
def test_datamap_with_three_headers(datamap_three_headers):
"""
Should pass because fourth col (type) is optional.
"""
data = datamap_reader(datamap_three_headers)
assert data[14].key == "Bad Spacing"
assert data[14].sheet == "Introduction"
assert data[14].cellref == "C35"
assert data[14].data_type is None
def test_datamap_with_only_single_header_raises_exception(
datamap_single_header):
"We want the datamap to be checked first and rejected if the headers are bad"
with pytest.raises(MalFormedCSVHeaderException) as excinfo:
data = datamap_reader(datamap_single_header) # noqa
msg = excinfo.value.args[0]
assert (
msg ==
"Datamap contains only one header - need at least three to proceed. Quitting."
)
def test_datamap_reader(mock_config, org_test_files_dir):
mock_config.initialise()
for fl in os.listdir(org_test_files_dir):
shutil.copy(
Path.cwd() / "tests" / "resources" / "org_templates" / fl,
(Path(mock_config.PLATFORM_DOCS_DIR) / "input"),
)
dm_file = mock_config.PLATFORM_DOCS_DIR / "input" / "dft_datamap.csv"
data = datamap_reader(dm_file)
assert data[0].key == "Project/Programme Name"
assert data[0].sheet == "Introduction"
assert data[0].cellref == "C11"
assert data[0].filename == str(dm_file)
def list_as_objs(self) -> List[DatamapLine]:
"""Return list of DatamapLine objects parsed from filepath."""
return datamap_reader(self.datamap_path)
def test_bad_spacing_in_datamap(datamap):
data = datamap_reader(datamap)
assert data[14].key == "Bad Spacing"
assert data[14].sheet == "Introduction"
assert data[14].cellref == "C35"
assert data[14].data_type == "TEXT"
def test_datamap_reader_supported_encodings(datamap_csv_supported_encodings):
data = datamap_reader(datamap_csv_supported_encodings)
assert data[0].key == "Project/Programme Name"
assert data[0].sheet == "Introduction"
assert data[0].cellref == "C11"
assert data[0].filename == str(datamap_csv_supported_encodings)
def test_datamap_reader_unsupported_encodings(
datamap_csv_unsupported_encodings):
with pytest.raises(DatamapFileEncodingError):
_ = datamap_reader(datamap_csv_unsupported_encodings)
def test_datamap_missing_key_fields(datamap_missing_key_fields):
"""We do not want the datamap read to pass if a line has a missing key field."""
with pytest.raises(MissingCellKeyError):
data = datamap_reader(datamap_missing_key_fields) # noqa
def test_datamap_missing_sheet_fields(datamap_missing_fields):
"""We do not want the datamap read to pass if a line has a missing sheet field."""
with pytest.raises(MissingSheetFieldError):
data = datamap_reader(datamap_missing_fields) # noqa
def test_datamap_with_seemingly_empty_rows_at_end_exception(
datamap_empty_tail_rows):
"A datamap file with multple empty rows at the end is possible, and unwelcome..."
with pytest.raises(MalFormedCSVEmptyTailRowsException):
data = datamap_reader(datamap_empty_tail_rows) # noqa
def test_datamap_file_as_pathlib_object(datamap):
data = datamap_reader(Path(datamap))
assert len(data) == 18
def test_very_bad_headers_are_rejected(datamap_very_bad_headers):
"We want the datamap to be checked first and rejected if the headers are bad"
with pytest.raises(MalFormedCSVHeaderException):
data = datamap_reader(datamap_very_bad_headers) # noqa
