def list_as_json(self) -> str: """Return list of DatamapLine objects parsed from filepath as json.""" try: lst_of_objs = datamap_reader(self.datamap_path) except DatamapNotCSVException: raise return json.dumps(lst_of_objs, cls=DatamapEncoder)
def check_aux_files(config: Config): """Check the presence of the blank template, the datamap and the validity of the datmap""" blank_t = check_for_blank(config) if blank_t[0]: logger.info( f"Blank template named {blank_t[1]} present in input directory.") else: logger.critical( f"No blank template present. Config requires a file called " f"{config.config_parser['DEFAULT']['blank file name']} in input directory." ) sys.exit(0) dm_t = check_for_datamap(config) if dm_t[0]: logger.info( f"Datamap file named {dm_t[1]} present in input directory.") else: logger.critical( f"No datamap file present. Config requires a file called " f"{config.config_parser['DEFAULT']['datamap file name']} in input directory." ) sys.exit(0) # if we get this far, there is a datamap file, so we can run this dm_name = config.config_parser["DEFAULT"]["datamap file name"] if datamap_reader(config.PLATFORM_DOCS_DIR / "input" / dm_name): logger.info( "Datamap file passes tests. Check any WARNING messages. Ok to proceed." ) else: logger.critical("Datamap tests failed")
def test_datamap_with_two_headers(datamap_two_headers): with pytest.raises(MalFormedCSVHeaderException) as excinfo: data = datamap_reader(datamap_two_headers) # noqa msg = excinfo.value.args[0] assert ( msg == "Datamap contains only two headers - need at least three to proceed. Quitting." )
def test_incorrect_headers_are_coerced_or_flagged( datamap_moderately_bad_headers): data = datamap_reader(datamap_moderately_bad_headers) # using same test as above because even though this datamap file has bad keys, we # accept them because they are not too bad... assert data[14].key == "Bad Spacing" assert data[14].sheet == "Introduction" assert data[14].cellref == "C35" assert data[14].data_type == "TEXT"
def test_datamap_type_is_optional(datamap_no_type_col): """We want the datamap to be processed even if it doesn't have a type col, which should be optional. """ data = datamap_reader(datamap_no_type_col) assert data[14].key == "Bad Spacing" assert data[14].sheet == "Introduction" assert data[14].cellref == "C35" assert data[14].data_type is None
def test_datamap_with_three_headers(datamap_three_headers): """ Should pass because fourth col (type) is optional. """ data = datamap_reader(datamap_three_headers) assert data[14].key == "Bad Spacing" assert data[14].sheet == "Introduction" assert data[14].cellref == "C35" assert data[14].data_type is None
def test_datamap_with_only_single_header_raises_exception( datamap_single_header): "We want the datamap to be checked first and rejected if the headers are bad" with pytest.raises(MalFormedCSVHeaderException) as excinfo: data = datamap_reader(datamap_single_header) # noqa msg = excinfo.value.args[0] assert ( msg == "Datamap contains only one header - need at least three to proceed. Quitting." )
def test_datamap_reader(mock_config, org_test_files_dir): mock_config.initialise() for fl in os.listdir(org_test_files_dir): shutil.copy( Path.cwd() / "tests" / "resources" / "org_templates" / fl, (Path(mock_config.PLATFORM_DOCS_DIR) / "input"), ) dm_file = mock_config.PLATFORM_DOCS_DIR / "input" / "dft_datamap.csv" data = datamap_reader(dm_file) assert data[0].key == "Project/Programme Name" assert data[0].sheet == "Introduction" assert data[0].cellref == "C11" assert data[0].filename == str(dm_file)
def list_as_objs(self) -> List[DatamapLine]: """Return list of DatamapLine objects parsed from filepath.""" return datamap_reader(self.datamap_path)
def test_bad_spacing_in_datamap(datamap): data = datamap_reader(datamap) assert data[14].key == "Bad Spacing" assert data[14].sheet == "Introduction" assert data[14].cellref == "C35" assert data[14].data_type == "TEXT"
def test_datamap_reader_supported_encodings(datamap_csv_supported_encodings): data = datamap_reader(datamap_csv_supported_encodings) assert data[0].key == "Project/Programme Name" assert data[0].sheet == "Introduction" assert data[0].cellref == "C11" assert data[0].filename == str(datamap_csv_supported_encodings)
def test_datamap_reader_unsupported_encodings( datamap_csv_unsupported_encodings): with pytest.raises(DatamapFileEncodingError): _ = datamap_reader(datamap_csv_unsupported_encodings)
def test_datamap_missing_key_fields(datamap_missing_key_fields): """We do not want the datamap read to pass if a line has a missing key field.""" with pytest.raises(MissingCellKeyError): data = datamap_reader(datamap_missing_key_fields) # noqa
def test_datamap_missing_sheet_fields(datamap_missing_fields): """We do not want the datamap read to pass if a line has a missing sheet field.""" with pytest.raises(MissingSheetFieldError): data = datamap_reader(datamap_missing_fields) # noqa
def test_datamap_with_seemingly_empty_rows_at_end_exception( datamap_empty_tail_rows): "A datamap file with multple empty rows at the end is possible, and unwelcome..." with pytest.raises(MalFormedCSVEmptyTailRowsException): data = datamap_reader(datamap_empty_tail_rows) # noqa
def test_datamap_file_as_pathlib_object(datamap): data = datamap_reader(Path(datamap)) assert len(data) == 18
def test_very_bad_headers_are_rejected(datamap_very_bad_headers): "We want the datamap to be checked first and rejected if the headers are bad" with pytest.raises(MalFormedCSVHeaderException): data = datamap_reader(datamap_very_bad_headers) # noqa