def key_family_dimensions_have_concept_and_code_list(): dsd_file = io.BytesIO(b"""<?xml version="1.0" encoding="UTF-8"?> <Structure xmlns="http://www.SDMX.org/resources/SDMXML/schemas/v2_0/message" xmlns:structure="http://www.SDMX.org/resources/SDMXML/schemas/v2_0/structure"> <Concepts> <structure:Concept id="COUNTRY" agencyID="OECD"> <structure:Name xml:lang="en">Country</structure:Name> </structure:Concept> </Concepts> <CodeList> <structure:CodeList id="CL_MON20123_2_COUNTRY" agencyID="OECD"> <structure:Name xml:lang="en">MON20123_2_COUNTRY codelist</structure:Name> </structure:CodeList> </CodeList> <KeyFamilies> <structure:KeyFamily id="MON20123_2" agencyID="OECD"> <structure:Name xml:lang="en">2012 F) OECD countries : Consumer Support Estimate by country</structure:Name> <structure:Components> <structure:Dimension conceptRef="COUNTRY" codelist="CL_MON20123_2_COUNTRY"/> </structure:Components> </structure:KeyFamily> </KeyFamilies> </Structure>""") dsd_reader = sdmx.dsd_reader(fileobj=dsd_file) key_family, = dsd_reader.key_families() dimension, = key_family.dimensions() assert_equal("COUNTRY", dimension.concept_ref()) assert_equal("CL_MON20123_2_COUNTRY", dimension.code_list_id())
def code_lists_are_indexed_by_id(): dsd_file = io.BytesIO(b"""<?xml version="1.0" encoding="UTF-8"?> <Structure xmlns="http://www.SDMX.org/resources/SDMXML/schemas/v2_0/message" xmlns:structure="http://www.SDMX.org/resources/SDMXML/schemas/v2_0/structure"> <CodeLists> <structure:CodeList id="CL_ITM_NEWA" agencyID="EUROSTAT" isFinal="true"> </structure:CodeList> </CodeLists> </Structure>""") dsd_reader = sdmx.dsd_reader(fileobj=dsd_file) assert_equal("CL_ITM_NEWA", dsd_reader.code_list("CL_ITM_NEWA").id) assert_equal(None, dsd_reader.code_list("CL_ITM_NEWB"))
def concepts_are_indexed_by_id(): dsd_file = io.BytesIO(b"""<?xml version="1.0" encoding="UTF-8"?> <Structure xmlns="http://www.SDMX.org/resources/SDMXML/schemas/v2_0/message" xmlns:structure="http://www.SDMX.org/resources/SDMXML/schemas/v2_0/structure"> <Concepts> <structure:Concept id="itm_newa"> <structure:Name xml:lang="en">List of products - EAA</structure:Name> </structure:Concept> </Concepts> </Structure>""") dsd_reader = sdmx.dsd_reader(fileobj=dsd_file) assert_equal("itm_newa", dsd_reader.concept("itm_newa").id) assert_equal(None, dsd_reader.concept("itm_newb"))
def concepts_are_detected_when_contained_directly_in_concepts_element(): dsd_file = io.BytesIO(b"""<?xml version="1.0" encoding="UTF-8"?> <Structure xmlns="http://www.SDMX.org/resources/SDMXML/schemas/v2_0/message" xmlns:structure="http://www.SDMX.org/resources/SDMXML/schemas/v2_0/structure"> <Concepts> <structure:Concept id="itm_newa"> <structure:Name xml:lang="en">List of products - EAA</structure:Name> </structure:Concept> </Concepts> </Structure>""") dsd_reader = sdmx.dsd_reader(fileobj=dsd_file) concept, = dsd_reader.concepts() assert_equal("itm_newa", concept.id)
def code_parent_is_treated_as_missing_if_empty_string(): dsd_file = io.BytesIO(b"""<?xml version="1.0" encoding="UTF-8"?> <Structure xmlns="http://www.SDMX.org/resources/SDMXML/schemas/v2_0/message" xmlns:structure="http://www.SDMX.org/resources/SDMXML/schemas/v2_0/structure"> <CodeLists> <structure:CodeList id="CL_ITM_NEWA" agencyID="EUROSTAT" isFinal="true"> <structure:Code value="40000" parentCode="" /> </structure:CodeList> </CodeLists> </Structure>""") dsd_reader = sdmx.dsd_reader(fileobj=dsd_file) code_list, = dsd_reader.code_lists() code, = code_list.codes() assert_equal(None, code.parent_code_id())
def code_lists_are_read_with_ids_and_names(): dsd_file = io.BytesIO(b"""<?xml version="1.0" encoding="UTF-8"?> <Structure xmlns="http://www.SDMX.org/resources/SDMXML/schemas/v2_0/message" xmlns:structure="http://www.SDMX.org/resources/SDMXML/schemas/v2_0/structure"> <CodeLists> <structure:CodeList id="CL_ITM_NEWA" agencyID="EUROSTAT" isFinal="true"> <structure:Name xml:lang="en">List of products - EAA</structure:Name> <structure:Name xml:lang="de">Liste der Produkte - LGR</structure:Name> </structure:CodeList> </CodeLists> </Structure>""") dsd_reader = sdmx.dsd_reader(fileobj=dsd_file) code_list, = dsd_reader.code_lists() assert_equal("CL_ITM_NEWA", code_list.id) assert_equal("List of products - EAA", code_list.name(lang="en")) assert_equal("Liste der Produkte - LGR", code_list.name(lang="de"))
def concept_names_are_read_by_language(): dsd_file = io.BytesIO(b"""<?xml version="1.0" encoding="UTF-8"?> <Structure xmlns="http://www.SDMX.org/resources/SDMXML/schemas/v2_0/message" xmlns:structure="http://www.SDMX.org/resources/SDMXML/schemas/v2_0/structure"> <Concepts> <structure:ConceptScheme> <structure:Concept id="itm_newa"><structure:Name xml:lang="en">List of products - EAA</structure:Name> <structure:Name xml:lang="de">Liste der Produkte - LGR</structure:Name> <structure:Name xml:lang="fr">Liste de produits - EAA</structure:Name> </structure:Concept> </structure:ConceptScheme> </Concepts> </Structure>""") dsd_reader = sdmx.dsd_reader(fileobj=dsd_file) concept, = dsd_reader.concepts() assert_equal("List of products - EAA", concept.name("en"))
def key_families_are_read_with_id_and_name(): dsd_file = io.BytesIO(b"""<?xml version="1.0" encoding="UTF-8"?> <Structure xmlns="http://www.SDMX.org/resources/SDMXML/schemas/v2_0/message" xmlns:structure="http://www.SDMX.org/resources/SDMXML/schemas/v2_0/structure"> <KeyFamilies> <structure:KeyFamily id="MON20123_2" agencyID="OECD"> <structure:Name xml:lang="en">2012 F) OECD countries : Consumer Support Estimate by country</structure:Name> <structure:Name xml:lang="fr">2012 F) OCDE : Estimation du soutien aux consommateurs par pays</structure:Name> </structure:KeyFamily> </KeyFamilies> </Structure>""") dsd_reader = sdmx.dsd_reader(fileobj=dsd_file) key_family, = dsd_reader.key_families() assert_equal("MON20123_2", key_family.id) assert_equal("2012 F) OECD countries : Consumer Support Estimate by country", key_family.name("en")) assert_equal("2012 F) OCDE : Estimation du soutien aux consommateurs par pays", key_family.name("fr"))
def primary_measure_can_be_read_from_key_family(): dsd_file = io.BytesIO(b"""<?xml version="1.0" encoding="UTF-8"?> <Structure xmlns="http://www.SDMX.org/resources/SDMXML/schemas/v2_0/message" xmlns:structure="http://www.SDMX.org/resources/SDMXML/schemas/v2_0/structure"> <KeyFamilies> <structure:KeyFamily id="MON20123_2" agencyID="OECD"> <structure:Name xml:lang="en">2012 F) OECD countries : Consumer Support Estimate by country</structure:Name> <structure:Components> <structure:PrimaryMeasure conceptRef="OBS_VALUE"><TextFormat textType="Double" /></structure:PrimaryMeasure> </structure:Components> </structure:KeyFamily> </KeyFamilies> </Structure>""") dsd_reader = sdmx.dsd_reader(fileobj=dsd_file) key_family, = dsd_reader.key_families() dimension = key_family.primary_measure() assert_equal("OBS_VALUE", dimension.concept_ref())
def time_dimension_can_be_read_from_key_family(): dsd_file = io.BytesIO(b"""<?xml version="1.0" encoding="UTF-8"?> <Structure xmlns="http://www.SDMX.org/resources/SDMXML/schemas/v2_0/message" xmlns:structure="http://www.SDMX.org/resources/SDMXML/schemas/v2_0/structure"> <KeyFamilies> <structure:KeyFamily id="MON20123_2" agencyID="OECD"> <structure:Name xml:lang="en">2012 F) OECD countries : Consumer Support Estimate by country</structure:Name> <structure:Components> <structure:TimeDimension conceptRef="TIME" codelist="CL_MON2012TSE_O_TIME" /> </structure:Components> </structure:KeyFamily> </KeyFamilies> </Structure>""") dsd_reader = sdmx.dsd_reader(fileobj=dsd_file) key_family, = dsd_reader.key_families() dimension = key_family.time_dimension() assert_equal("TIME", dimension.concept_ref()) assert_equal("CL_MON2012TSE_O_TIME", dimension.code_list_id())
def codes_can_be_read_from_code_lists(): dsd_file = io.BytesIO(b"""<?xml version="1.0" encoding="UTF-8"?> <Structure xmlns="http://www.SDMX.org/resources/SDMXML/schemas/v2_0/message" xmlns:structure="http://www.SDMX.org/resources/SDMXML/schemas/v2_0/structure"> <CodeLists> <structure:CodeList id="CL_ITM_NEWA" agencyID="EUROSTAT" isFinal="true"> <structure:Code value="40000"> <structure:Description xml:lang="en">Total labour force input</structure:Description> <structure:Description xml:lang="de">Landwirtschaftlicher Arbeitseinsatz insgesamt</structure:Description> </structure:Code> </structure:CodeList> </CodeLists> </Structure>""") dsd_reader = sdmx.dsd_reader(fileobj=dsd_file) code_list, = dsd_reader.code_lists() code, = code_list.codes() assert_equal("40000", code.value) assert_equal(None, code.parent_code_id()) assert_equal("Total labour force input", code.description("en")) assert_equal("Landwirtschaftlicher Arbeitseinsatz insgesamt", code.description("de"))
def get_dataset_structure(self, database, dataset) -> Dataset: """ Obtain the structure of a dataset: concepts, dimensions, attributes and measures """ # Retrieve and save to file xml = requests.get( "http://stats.oecd.org/restsdmx/sdmx.ashx/GetDataStructure/" + dataset) xml = xml.content.decode("utf-8") _, fname = tempfile.mkstemp(".oecd_dsd.xml", text=True) file = open(fname, "w") file.writelines(xml) file.close() dsd = sdmx.dsd_reader(fname) kfam = dsd.key_families()[0] # SDMXConcept = collections.namedtuple('Concept', 'type name istime description code_list') # DataSource <- Database <- DATASET <- Dimension(s) (including Measures) <- CodeList # | # v # Concept <- CodeList (NOT CONSIDERED NOW) ds = Dataset() ds.code = dataset ds.description = kfam.name("en") ds.attributes = { } # Dataset level attributes? (encode them using a dictionary) ds.metadata = None # Metadata for the dataset SDMX (flow, date of production, etc.) ds.database = database # Reference to containing database # Code lists (assume flat) c_dict = {} for cl in dsd.code_lists(): codes = [ CodeImmutable(c.value, c.description("en"), "", []) for c in cl.codes() ] c_dict[cl.id] = CodeList.construct(cl.id, cl.name("en"), [""], codes=codes) # Time dimension name if kfam.time_dimension(): t_dim = kfam.time_dimension().concept_ref() t_dim_cl = kfam.time_dimension().code_list_id() else: t_dim = None # Dimensions for d in kfam.dimensions(): dd = Dimension() dd.code = d.concept_ref() dd.attributes = None istime = dd.code == t_dim if istime: dd.is_time = istime dd.code_list = c_dict[t_dim_cl] else: dd.code_list = c_dict[d.code_list_id()] dd.description = dd.code_list.description dd.is_measure = False # DIMENSION dd.dataset = ds # Measure dd = Dimension() dd.code = kfam.primary_measure().concept_ref() dd.description = dsd.concept(dd.code).name("en") dd.attributes = None dd.is_time = False dd.is_measure = True dd.dataset = ds return ds