def test_definitions_loaded_for_columns_not_in_data(): """ Ensure that column definitions from an info.json file as loaded even if the column isn't defined in the initial loaded data. This is primarily to ensure that cube validation highlights that they've defined columns which aren't in the data. """ data = pd.read_csv(get_test_cases_dir() / "configloaders" / "multi-measure-multi-unit-test-files" / "multi-measure-unit-data.csv") del data["Unit"] # delete a column from the data cube, json_schema_validation_error = get_cube_from_info_json( get_test_cases_dir() / "configloaders" / "multi-measure-multi-unit-test-files" / "multi-measure-unit-info.json", data, ) errors = cube.validate() assert_num_validation_errors(errors, 1) error = errors[0] assert isinstance(error, ColumnNotFoundInDataError) assert error.csv_column_title == "Unit"
def test_single_measure_obs_val_unit_measure_data_type(): """ Test that the datatype, unit & measure are correctly extracted from a single-measure dataset's info.json file """ data = pd.read_csv(get_test_cases_dir() / "configloaders" / "data.csv") cube, json_schema_validation_error = get_cube_from_info_json( get_test_cases_dir() / "configloaders" / "info.json", data) obs_val_cols = get_columns_of_dsd_type(cube, QbSingleMeasureObservationValue) assert len(obs_val_cols) == 1 obs_val = obs_val_cols[0].structural_definition assert isinstance(obs_val, QbSingleMeasureObservationValue) unit = obs_val.unit assert unit is not None assert isinstance(unit, ExistingQbUnit) assert (unit.unit_uri == "http://gss-data.org.uk/def/concept/measurement-units/gbp-million") measure = obs_val.measure assert isinstance(measure, ExistingQbMeasure) assert measure.measure_uri == "http://gss-data.org.uk/def/measure/trade" assert obs_val.data_type == "double"
def test_multiple_measures_and_units_loaded_in_uri_template(): """ multi-measure-unit-data.csv has multiple measures and multiple units The JSON schema for the info.json files which defines all of the possible properties an info.json can have is to be found at <https://github.com/GSS-Cogs/family-schemas/blob/main/dataset-schema.json>. """ data = pd.read_csv(get_test_cases_dir() / "configloaders" / "multi-measure-multi-unit-test-files" / "multi-measure-unit-data.csv") cube, json_schema_validation_error = get_cube_from_info_json( get_test_cases_dir() / "configloaders" / "multi-measure-multi-unit-test-files" / "multi-measure-unit-info.json", data, ) """Measure URI""" expected_measure_uris = [ "http://gss-data.org.uk/def/x/one-litre-and-less", "http://gss-data.org.uk/def/x/more-than-one-litre", "http://gss-data.org.uk/def/x/number-of-bottles", ] measure_column = cube.columns[1] assert type(measure_column) == QbColumn assert type( measure_column.structural_definition) == QbMultiMeasureDimension # # [str(c) for c in cube.columns] actual_measure_uris = [ x.measure_uri for x in measure_column.structural_definition.measures ] assert len(expected_measure_uris) == len(actual_measure_uris) assert not (set(expected_measure_uris) ^ set(actual_measure_uris)) # """Unit URI""" unit_column = cube.columns[2] assert type(unit_column) == QbColumn assert type(unit_column.structural_definition) == QbMultiUnits expected_unit_uris = [ "http://gss-data.org.uk/def/concept/measurement-units/count", "http://gss-data.org.uk/def/concept/measurement-units/percentage", ] actual_unit_uris = [ x.unit_uri for x in unit_column.structural_definition.units ] assert len(expected_unit_uris) == len(actual_unit_uris) assert not (set(expected_unit_uris) ^ set(actual_unit_uris)) errors = cube.validate() errors += validate_qb_component_constraints(cube) assert_num_validation_errors(errors, 0)
def test_units_loading(): """ Test that multi-units columns with existing and new units can be successfully loaded using the new info.json v1.1 syntax using all possible configuration types. """ data = pd.read_csv(get_test_cases_dir() / "configloaders" / "infojson1-1" / "units.csv") cube, json_schema_validation_error = get_cube_from_info_json( get_test_cases_dir() / "configloaders" / "infojson1-1" / "units.json", data, ) existing_units = _get_column_definition(cube, "Existing Units") new_units = _get_column_definition(cube, "New Units") new_units_2 = _get_column_definition(cube, "New Units 2") new_units_3 = _get_column_definition(cube, "New Units 3") assert existing_units is not None assert isinstance(existing_units.structural_definition, QbMultiUnits) assert len(existing_units.structural_definition.units) == 1 unit = existing_units.structural_definition.units[0] assert isinstance(unit, ExistingQbUnit) assert unit.unit_uri == "http://example.com/units/some-existing-unit" assert (existing_units.csv_column_uri_template == "http://example.com/units/{+existing_units}") assert new_units is not None assert isinstance(new_units.structural_definition, QbMultiUnits) assert len(new_units.structural_definition.units) == 1 unit = new_units.structural_definition.units[0] assert isinstance(unit, NewQbUnit) assert unit.label == "New Unit 1" assert new_units.csv_column_uri_template is None assert new_units_2 is not None assert isinstance(new_units_2.structural_definition, QbMultiUnits) assert len(new_units_2.structural_definition.units) == 1 unit = new_units_2.structural_definition.units[0] assert isinstance(unit, NewQbUnit) assert unit.label == "New Unit 2" assert unit.uri_safe_identifier_override == "newunit-2" assert unit.description == "New Unit 2 Comment" assert unit.source_uri == "http://example.com/units/defs/new-unit-2" assert isinstance(unit.base_unit, ExistingQbUnit) assert (unit.base_unit.unit_uri == "http://gss-data.org.uk/def/concept/measurement-units/some-unit") assert unit.base_unit_scaling_factor == 7 assert (unit.qudt_quantity_kind_uri == "http://qudt.org/vocab/quantitykind/AbsoluteHumidity") assert unit.si_base_unit_conversion_multiplier == 200 assert new_units_2.csv_column_uri_template is None assert new_units_3 is not None assert isinstance(new_units_3.structural_definition, QbMultiUnits) assert len(new_units_3.structural_definition.units) == 1 unit = new_units_3.structural_definition.units[0] assert isinstance(unit, NewQbUnit) assert unit.label == "New Unit 3" assert new_units_3.csv_column_uri_template is None
def test_measures_loading(): """ Test that multi-measure dimensions with existing and new measures can be successfully loaded using the new info.json v1.1 syntax using all possible configuration types. """ data = pd.read_csv(get_test_cases_dir() / "configloaders" / "infojson1-1" / "measures.csv") cube, json_schema_validation_error = get_cube_from_info_json( get_test_cases_dir() / "configloaders" / "infojson1-1" / "measures.json", data, ) existing_measures = _get_column_definition(cube, "Existing Measures") new_measures = _get_column_definition(cube, "New Measures") new_measures_2 = _get_column_definition(cube, "New Measures 2") new_measures_3 = _get_column_definition(cube, "New Measures 3") assert existing_measures is not None assert isinstance(existing_measures.structural_definition, QbMultiMeasureDimension) assert len(existing_measures.structural_definition.measures) == 1 measure = existing_measures.structural_definition.measures[0] assert isinstance(measure, ExistingQbMeasure) assert measure.measure_uri == "http://example.com/measures/some-existing-measure" assert (existing_measures.csv_column_uri_template == "http://example.com/measures/{+existing_measures}") assert new_measures is not None assert isinstance(new_measures.structural_definition, QbMultiMeasureDimension) assert len(new_measures.structural_definition.measures) == 1 measure = new_measures.structural_definition.measures[0] assert isinstance(measure, NewQbMeasure) assert measure.label == "Some Measure 1" assert new_measures.csv_column_uri_template is None assert new_measures_2 is not None assert isinstance(new_measures_2.structural_definition, QbMultiMeasureDimension) assert len(new_measures_2.structural_definition.measures) == 1 measure = new_measures_2.structural_definition.measures[0] assert isinstance(measure, NewQbMeasure) assert measure.label == "Some Measure 2" assert measure.description == "Some Measure 2 Comment" assert measure.uri_safe_identifier_override == "some-measure-2" assert measure.source_uri == "http://example.com/measures/defs/some-measure-2" assert new_measures_2.csv_column_uri_template is None assert new_measures_3 is not None assert isinstance(new_measures_3.structural_definition, QbMultiMeasureDimension) assert len(new_measures_3.structural_definition.measures) == 1 measure = new_measures_3.structural_definition.measures[0] assert isinstance(measure, NewQbMeasure) assert measure.label == "Some Measure 3" assert new_measures_3.csv_column_uri_template is None
def build( info_json: Path, catalog_metadata_json_file: Optional[Path], output_directory: Path, csv_path: Path, fail_when_validation_error_occurs: bool, validation_errors_file_out: Optional[Path], ): print(f"{Style.DIM}CSV: {csv_path.absolute()}") print(f"{Style.DIM}info.json: {info_json.absolute()}") data = pd.read_csv(csv_path) assert isinstance(data, pd.DataFrame) cube, json_schema_validation_errors = get_cube_from_info_json(info_json, data) if catalog_metadata_json_file is not None: _override_catalog_metadata_state(catalog_metadata_json_file, cube) validation_errors = cube.validate() validation_errors += validate_qb_component_constraints(cube) if not output_directory.exists(): print(f"Creating output directory {output_directory.absolute()}") output_directory.mkdir() if len(validation_errors) > 0 or len(json_schema_validation_errors) > 0: for error in validation_errors: print(f"{Fore.RED + Style.BRIGHT}Validation Error: {Style.NORMAL + error.message}") if isinstance(error, SpecificValidationError): print(f"More information: {error.get_error_url()}") for err in json_schema_validation_errors: print( f"{Fore.LIGHTRED_EX + Style.BRIGHT}Schema Validation Error: {Style.NORMAL + err.message}" ) if validation_errors_file_out is not None: validation_errors_dict = [ e.as_json_dict() if isinstance(e, DataClassBase) else dataclasses.asdict(e) for e in validation_errors ] all_errors = validation_errors_dict + [ e.message for e in json_schema_validation_errors ] with open(validation_errors_file_out, "w+") as f: json.dump(all_errors, f, indent=4) if fail_when_validation_error_occurs and len(validation_errors) > 0: exit(1) qb_writer = QbWriter(cube) qb_writer.write(output_directory) print(f"{Fore.GREEN + Style.BRIGHT}Build Complete")
def step_impl(context, some_json, some_csv): tmp_dir = get_context_temp_dir_path(context) data = pd.read_csv(tmp_dir / some_csv) cube_value, json_schema_errors = infojsonloader.get_cube_from_info_json( tmp_dir / some_json, data, ) context.cube = cube_value context.json_schema_errors = json_schema_errors
def test_csv_cols_assumed_dimensions(): """ If a column isn't defined, assume it is a new local dimension. Assume that if a column isn't defined in the info.json `transform.columns` section, then it is a new locally defined dimension. Assert that the newly defined dimension has a codelist created from the values in the CSV. """ data = pd.read_csv(get_test_cases_dir() / "configloaders" / "data.csv") cube, json_schema_validation_error = get_cube_from_info_json( get_test_cases_dir() / "configloaders" / "info.json", data) matching_columns = [ c for c in cube.columns if c.csv_column_title == "Undefined Column" ] assert len(matching_columns) == 1 undefined_column_assumed_definition: CsvColumn = matching_columns[0] if not isinstance(undefined_column_assumed_definition, QbColumn): raise Exception("Incorrect type") assert type(undefined_column_assumed_definition.structural_definition ) == NewQbDimension new_dimension: NewQbDimension = undefined_column_assumed_definition.structural_definition assert new_dimension.code_list is not None if not isinstance(new_dimension.code_list, NewQbCodeList): raise Exception("Incorrect type") newly_defined_concepts = list(new_dimension.code_list.concepts) assert len(newly_defined_concepts) == 1 new_concept = newly_defined_concepts[0] assert "Undefined Column Value" == new_concept.label errors = cube.validate() errors += validate_qb_component_constraints(cube) assert_num_validation_errors(errors, 0)
def test_cube_metadata_extracted_from_info_json(): """Metadata - ['base_uri', 'creator', 'description', 'from_dict', 'issued', 'keywords', 'landing_page', 'license', 'public_contact_point', 'publisher', 'summary', 'themes', 'title', 'uri_safe_identifier', 'validate']""" data = pd.read_csv(get_test_cases_dir() / "configloaders" / "multi-measure-multi-unit-test-files" / "multi-measure-unit-data.csv") cube, json_schema_validation_error = get_cube_from_info_json( get_test_cases_dir() / "configloaders" / "multi-measure-multi-unit-test-files" / "multi-measure-unit-info.json", data, ) # Creator - pass expected_creator = "https://www.gov.uk/government/organisations/hm-revenue-customs" actual_creator = cube.metadata.creator_uri assert expected_creator == actual_creator # Description - pass expected_description = ( "All bulletins provide details on percentage of one litre or less & more than " "one litre bottles. This information is provided on a yearly basis.") actual_description = cube.metadata.description assert expected_description == actual_description # issue_date - pass expected_issued_date = parser.parse("2019-02-28") actual_issued_date = cube.metadata.dataset_issued assert actual_issued_date == expected_issued_date # keywords - pass # There's currently no `keywords` property to map from the info.json. expected_keywords = [] actual_keywords = cube.metadata.keywords assert len(expected_keywords) == len(actual_keywords) assert not (set(expected_keywords) ^ set(actual_keywords)) # landingpage - pass expected_landingpage = [ "https://www.gov.uk/government/statistics/bottles-bulletin" ] actual_landingpage = cube.metadata.landing_page_uris assert expected_landingpage == actual_landingpage # license - pass # Surprisingly the info.json schema doesn't allow a licence property just yet. expected_license = None actual_license = cube.metadata.license_uri assert expected_license == actual_license # public_contact_point - pass # The info.json schema doesn't allow a public_contact_point property just yet expected_public_contact_point = None actual_public_contact_point = cube.metadata.public_contact_point_uri assert expected_public_contact_point == actual_public_contact_point # publisher - pass expected_publisher = ( "https://www.gov.uk/government/organisations/hm-revenue-customs") actual_publisher = cube.metadata.publisher_uri assert expected_publisher == actual_publisher # summary - pass # The info.json schema doesn't allow a summary property just yet expected_summary = None actual_summary = cube.metadata.summary assert expected_summary == actual_summary # themes - pass # It's the families property expected_themes = ["http://gss-data.org.uk/def/gdp#trade"] actual_themes = [str(t) for t in cube.metadata.theme_uris] assert len(expected_themes) == len(actual_themes) assert not (set(expected_themes) ^ set(actual_themes)) # title - pass expected_title = "bottles" actual_title = cube.metadata.title assert expected_title == actual_title # uri_safe_identifier - pass expected_uri_safe_identifier = "bottles-bulletin" actual_uri_safe_identifier = cube.metadata.uri_safe_identifier assert expected_uri_safe_identifier == actual_uri_safe_identifier errors = cube.validate() errors += validate_qb_component_constraints(cube) assert_num_validation_errors(errors, 0)
def test_attribute_loading(): """ Test that existing and new attributes can be successfully loaded using the new info.json v1.1 syntax using all possible configuration types. """ data = pd.read_csv(info_json_1_1_test_cases_dir / "attributes.csv") cube, json_schema_validation_error = get_cube_from_info_json( info_json_1_1_test_cases_dir / "attributes.json", data, ) existing_marker = _get_column_definition(cube, "Existing Marker") existing_marker_2 = _get_column_definition(cube, "Existing Marker 2") existing_marker_3 = _get_column_definition(cube, "Existing Marker 3") new_marker = _get_column_definition(cube, "New Marker") new_marker_2 = _get_column_definition(cube, "New Marker 2") new_marker_3 = _get_column_definition(cube, "New Marker 3") new_marker_4 = _get_column_definition(cube, "New Marker 4") new_marker_5 = _get_column_definition(cube, "New Marker 5") existing_attribute_literal = _get_column_definition( cube, "Existing Attribute Literal") new_attribute_literal = _get_column_definition(cube, "New Attribute Literal") assert existing_marker is not None assert isinstance(existing_marker.structural_definition, ExistingQbAttribute) assert (existing_marker.structural_definition.attribute_uri == "http://purl.org/linked-data/sdmx/2009/attribute#obsStatus") assert (existing_marker.csv_column_uri_template == "http://gss-data.org.uk/def/concept/cogs-markers/{marker}") assert len(existing_marker.structural_definition.new_attribute_values) == 0 assert existing_marker.structural_definition.is_required assert existing_marker_2 is not None assert isinstance(existing_marker_2.structural_definition, ExistingQbAttribute) assert (existing_marker_2.structural_definition.attribute_uri == "http://purl.org/linked-data/sdmx/2009/attribute#obsStatus") assert len( existing_marker_2.structural_definition.new_attribute_values) == 1 assert (existing_marker_2.structural_definition.new_attribute_values[0]. label == "Some Marker Value") assert not existing_marker_2.structural_definition.is_required assert existing_marker_2.csv_column_uri_template is None assert existing_marker_3 is not None assert isinstance(existing_marker_3.structural_definition, ExistingQbAttribute) assert (existing_marker_3.structural_definition.attribute_uri == "http://purl.org/linked-data/sdmx/2009/attribute#obsStatus") assert len( existing_marker_3.structural_definition.new_attribute_values) == 1 assert (existing_marker_3.structural_definition.new_attribute_values[0]. label == "Some Incorrect Marker Value") assert not existing_marker_3.structural_definition.is_required assert existing_marker_3.csv_column_uri_template is None assert new_marker is not None assert isinstance(new_marker.structural_definition, NewQbAttribute) assert new_marker.structural_definition.label == "New Marker" assert len(new_marker.structural_definition.new_attribute_values) == 1 assert new_marker.structural_definition.new_attribute_values[ 0].label == "Some Marker Value" assert new_marker.structural_definition.is_required assert new_marker.csv_column_uri_template is None assert new_marker_2 is not None assert isinstance(new_marker_2.structural_definition, NewQbAttribute) assert new_marker_2.structural_definition.label == "New Marker 2" assert new_marker_2.structural_definition.uri_safe_identifier_override == "new-marker-2" assert new_marker_2.structural_definition.description == "This is new marker 2" assert (new_marker_2.structural_definition.parent_attribute_uri == "http://gss-data.org.uk/def/trade/property/attribute/marker") assert (new_marker_2.structural_definition.source_uri == "http://example.org/attributes/markers/new-marker-2") assert len(new_marker_2.structural_definition.new_attribute_values) == 1 assert new_marker_2.structural_definition.new_attribute_values[ 0].label == "Some Marker Value" assert not new_marker_2.structural_definition.is_required assert new_marker_2.csv_column_uri_template is None assert new_marker_3 is not None assert isinstance(new_marker_3.structural_definition, NewQbAttribute) assert new_marker_3.structural_definition.label == "New Marker 3" assert len(new_marker_3.structural_definition.new_attribute_values) == 1 assert (new_marker_3.structural_definition.new_attribute_values[0].label == "Some Incorrect Marker Value") assert not new_marker_3.structural_definition.is_required assert new_marker_3.csv_column_uri_template is None assert new_marker_4 is not None assert isinstance(new_marker_4.structural_definition, NewQbAttribute) assert new_marker_4.structural_definition.label == "New Marker 4" assert len(new_marker_4.structural_definition.new_attribute_values) == 0 assert not new_marker_4.structural_definition.is_required assert (new_marker_4.csv_column_uri_template == "http://example.org/attributes/new-marker-4/{+new_marker_4}") assert new_marker_5 is not None assert isinstance(new_marker_5.structural_definition, NewQbAttribute) assert new_marker_5.structural_definition.label == "New Marker 5" assert len(new_marker_5.structural_definition.new_attribute_values) == 1 assert new_marker_5.structural_definition.new_attribute_values[ 0].label == "Some Marker Value" assert not new_marker_5.structural_definition.is_required assert new_marker_5.csv_column_uri_template is None assert existing_attribute_literal is not None assert isinstance(existing_attribute_literal.structural_definition, ExistingQbAttributeLiteral) assert ( existing_attribute_literal.structural_definition.attribute_uri == "http://example.com/attributes/with-literals/some-existing-attribute-with-literals" ) assert not existing_attribute_literal.structural_definition.is_required assert existing_attribute_literal.structural_definition.data_type == "double" assert new_attribute_literal is not None assert isinstance(new_attribute_literal.structural_definition, NewQbAttributeLiteral) assert new_attribute_literal.structural_definition.new_attribute_values == [] assert not new_attribute_literal.structural_definition.is_required assert new_attribute_literal.structural_definition.data_type == "double"
def test_dimension_loading(): """ Test that existing and new dimensions can be successfully loaded using the new info.json v1.1 syntax using all possible configuration types. """ data = pd.read_csv(info_json_1_1_test_cases_dir / "dimensions.csv") cube, json_schema_validation_error = get_cube_from_info_json( info_json_1_1_test_cases_dir / "dimensions.json", data, ) existing_dimension = _get_column_definition(cube, "Existing Dimension") new_dimension = _get_column_definition(cube, "New Dimension") new_dimension_2 = _get_column_definition(cube, "New Dimension 2") new_dimension_3 = _get_column_definition(cube, "New Dimension 3") new_dimension_4 = _get_column_definition(cube, "New Dimension 4") new_dimension_5 = _get_column_definition(cube, "New Dimension 5") new_dimension_6 = _get_column_definition(cube, "New Dimension 6") new_dimension_7 = _get_column_definition(cube, "New Dimension 7") assert existing_dimension is not None assert isinstance(existing_dimension.structural_definition, ExistingQbDimension) assert (existing_dimension.structural_definition.dimension_uri == "http://purl.org/linked-data/sdmx/2009/dimension#refPeriod") assert (existing_dimension.csv_column_uri_template == "http://reference.data.gov.uk/id/{period}") assert new_dimension is not None assert isinstance(new_dimension.structural_definition, NewQbDimension) assert new_dimension.structural_definition.label == "New Dimension" assert isinstance(new_dimension.structural_definition.code_list, NewQbCodeList) assert len(new_dimension.structural_definition.code_list.concepts) == 1 assert new_dimension.structural_definition.code_list.concepts[ 0].label == "Some Value" assert new_dimension.csv_column_uri_template is None assert new_dimension_2 is not None assert isinstance(new_dimension_2.structural_definition, NewQbDimension) assert new_dimension_2.structural_definition.label == "New Dimension 2 Label" assert new_dimension_2.structural_definition.uri_safe_identifier_override == "new-dimension-2" assert new_dimension_2.structural_definition.description == "New Dimension 2 Comment" assert (new_dimension_2.structural_definition.parent_dimension_uri == "http://purl.org/linked-data/sdmx/2009/dimension#refPeriod") assert new_dimension_2.structural_definition.code_list is None assert (new_dimension_2.csv_column_uri_template == "http://reference.data.gov.uk/id/{period}") assert new_dimension_3 is not None assert isinstance(new_dimension_3.structural_definition, NewQbDimension) assert new_dimension_3.structural_definition.label == "New Dimension 3" assert isinstance(new_dimension_3.structural_definition.code_list, NewQbCodeList) assert len(new_dimension_3.structural_definition.code_list.concepts) == 1 assert new_dimension_3.structural_definition.code_list.concepts[ 0].label == "Some Value" assert new_dimension_3.csv_column_uri_template is None assert new_dimension_4 is not None assert isinstance(new_dimension_4.structural_definition, NewQbDimension) assert new_dimension_4.structural_definition.label == "New Dimension 4" assert isinstance(new_dimension_4.structural_definition.code_list, ExistingQbCodeList) assert (new_dimension_4.structural_definition.code_list.concept_scheme_uri == "http://data.europa.eu/nuts") assert (new_dimension_4.csv_column_uri_template == "http://data.europa.eu/nuts/{+new_dimension_4}") assert new_dimension_5 is not None assert isinstance(new_dimension_5.structural_definition, NewQbDimension) assert new_dimension_5.structural_definition.label == "New Dimension 5" assert isinstance(new_dimension_5.structural_definition.code_list, NewQbCodeListInCsvW) assert (new_dimension_5.structural_definition.code_list. schema_metadata_file_path == info_json_1_1_test_cases_dir / "codelists" / "new-dimension-5.csv-metadata.json") assert new_dimension_5.csv_column_uri_template is None assert new_dimension_6 is not None assert isinstance(new_dimension_6.structural_definition, NewQbDimension) assert new_dimension_6.structural_definition.label == "New Dimension 6" assert isinstance(new_dimension_6.structural_definition.code_list, NewQbCodeList) assert len(new_dimension_6.structural_definition.code_list.concepts) == 1 assert new_dimension_6.structural_definition.code_list.concepts[ 0].label == "Some Value" assert new_dimension_6.csv_column_uri_template is None assert new_dimension_7 is not None assert isinstance(new_dimension_7.structural_definition, NewQbDimension) assert new_dimension_7.structural_definition.label == "New Dimension 7" assert isinstance(new_dimension_7.structural_definition.code_list, CompositeQbCodeList) assert len(new_dimension_7.structural_definition.code_list.concepts) == 1 code_list = new_dimension_7.structural_definition.code_list assert len(code_list.variant_of_uris) == 0 first_concept = code_list.concepts[0] assert (first_concept.existing_concept_uri == "http://reference.data.gov.uk/id/year/2018") assert first_concept.label == "2018" assert new_dimension_7.csv_column_uri_template is None
def test_observation_value_loading(): """ Test that single and multi-measure observation value columns can be successfully loaded using the new info.json v1.1 syntax using all possible configuration types. """ data = pd.read_csv(info_json_1_1_test_cases_dir / "observations.csv") cube, json_schema_validation_error = get_cube_from_info_json( info_json_1_1_test_cases_dir / "observations.json", data, ) single_measure = _get_column_definition(cube, "Single Measure Value") single_measure_2 = _get_column_definition(cube, "Single Measure Value 2") multi_measure = _get_column_definition(cube, "Multi Measure Value") multi_measure_2 = _get_column_definition(cube, "Multi Measure Value 2") assert single_measure is not None assert isinstance(single_measure.structural_definition, QbSingleMeasureObservationValue) assert isinstance(single_measure.structural_definition.measure, ExistingQbMeasure) assert (single_measure.structural_definition.measure.measure_uri == "http://gss-data.org.uk/def/measure/trade") assert isinstance(single_measure.structural_definition.unit, ExistingQbUnit) assert (single_measure.structural_definition.unit.unit_uri == "http://gss-data.org.uk/def/concept/measurement-units/gbp-million") assert single_measure.structural_definition.data_type == "double" assert single_measure.csv_column_uri_template is None assert single_measure_2 is not None assert isinstance(single_measure_2.structural_definition, QbSingleMeasureObservationValue) assert isinstance(single_measure_2.structural_definition.measure, NewQbMeasure) assert single_measure_2.structural_definition.measure.label == "New Measure 2" assert isinstance(single_measure_2.structural_definition.unit, NewQbUnit) assert single_measure_2.structural_definition.unit.label == "New Unit 2" assert isinstance(single_measure_2.structural_definition.unit.base_unit, ExistingQbUnit) assert (single_measure_2.structural_definition.unit.base_unit.unit_uri == "http://gss-data.org.uk/def/concept/measurement-units/some-unit") assert single_measure_2.structural_definition.unit.base_unit_scaling_factor == 7 assert single_measure_2.structural_definition.data_type == "decimal" assert single_measure_2.csv_column_uri_template is None assert multi_measure is not None assert isinstance(multi_measure.structural_definition, QbMultiMeasureObservationValue) assert isinstance(multi_measure.structural_definition.unit, ExistingQbUnit) assert ( multi_measure.structural_definition.unit.unit_uri == "http://gss-data.org.uk/def/concept/measurement-units/gbp-billions") assert multi_measure.structural_definition.data_type == "decimal" assert multi_measure.csv_column_uri_template is None assert multi_measure_2 is not None assert isinstance(multi_measure_2.structural_definition, QbMultiMeasureObservationValue) assert multi_measure_2.structural_definition.unit is None assert multi_measure_2.structural_definition.data_type == "short" assert multi_measure_2.csv_column_uri_template is None