def test_refcount(): # Component (subclasses) created outside of a DataStructureDefinition da1 = DataAttribute(id="foo") da2 = DataAttribute(id="foo") assert id(da1) != id(da2) assert da1 is not da2 # Retrieving attributes from a DataStructureDefinition results in references # to the same object dsd = DataStructureDefinition() da3 = dsd.attributes.getdefault("foo") da4 = dsd.attributes.getdefault("foo") assert id(da3) == id(da4) assert da3 is da4 assert len(dsd.attributes) == 1 # Creating an AttributeValue referencing a DataAttribute outside a DSD av1 = AttributeValue(value="baz", value_for=DataAttribute(id="foo")) assert not any([av1.value_for is da for da in (da1, da2, da3)]) # Same, using a DSD av2 = AttributeValue(value="baz", value_for="foo", dsd=dsd) assert av2.value_for is da3
def test_observation(): obs = Observation() # Set by item name obs.attached_attribute["TIME_PERIOD"] = 3 # NB the following does not work; see Observation.attrib() # obs.attrib['TIME_PERIOD'] = 3 obs.attached_attribute["CURRENCY"] = "USD" # Access by attribute name assert obs.attrib.TIME_PERIOD == 3 assert obs.attrib.CURRENCY == "USD" # Access by item index assert obs.attrib[1] == "USD" # Add attributes obs.attached_attribute["FOO"] = "1" obs.attached_attribute["BAR"] = "2" assert obs.attrib.FOO == "1" and obs.attrib["BAR"] == "2" # Using classes da = DataAttribute(id="FOO") av = AttributeValue(value_for=da, value="baz") obs.attached_attribute[da.id] = av assert obs.attrib[da.id] == "baz"
def get_attribute_values(self, row, indicator, relationship_type): values = {} for attribute in self.dsd.attributes: valid_attribute = False if relationship_type == 'series' and isinstance(attribute.related_to, DimensionRelationship): valid_attribute = True elif relationship_type == 'observation' and attribute.related_to is PrimaryMeasureRelationship: valid_attribute = True if valid_attribute: value = row[attribute.id] if attribute.id in row else self.get_attribute_default(attribute.id, indicator) if value != '': values[attribute.id] = AttributeValue(value_for=attribute, value=str(value)) return values
def test_flat(): # Create a bare Message msg = DataMessage() # Recreate the content from exr-flat.json header = Header( id="62b5f19d-f1c9-495d-8446-a3661ed24753", prepared="2012-11-29T08:40:26Z", sender=model.Agency(id="ECB"), ) msg.header = header ds = DataSet() # Create a Key and attributes key = Key( FREQ="D", CURRENCY="NZD", CURRENCY_DENOM="EUR", EXR_TYPE="SP00", EXR_SUFFIX="A", TIME_PERIOD="2013-01-18", ) obs_status = DataAttribute(id="OBS_STATUS") attr = {"OBS_STATUS": AttributeValue(value_for=obs_status, value="A")} ds.obs.append( Observation(dimension=key, value=1.5931, attached_attribute=attr)) key = key.copy(TIME_PERIOD="2013-01-21") ds.obs.append( Observation(dimension=key, value=1.5925, attached_attribute=attr)) key = key.copy(CURRENCY="RUB", TIME_PERIOD="2013-01-18") ds.obs.append( Observation(dimension=key, value=40.3426, attached_attribute=attr)) key = key.copy(TIME_PERIOD="2013-01-21") ds.obs.append( Observation(dimension=key, value=40.3000, attached_attribute=attr)) msg.data.append(ds) # Write to pd.Dataframe df1 = sdmx.to_pandas(msg) with specimen("flat.json") as f: ref = sdmx.read_sdmx(f) df2 = sdmx.to_pandas(ref) assert_pd_equal(df1, df2)
def test_add_obs(DataSetType): # Create a Key and Attributes key = Key(CURRENCY="NZD", CURRENCY_DENOM="EUR", TIME_PERIOD="2018-01-01") obs_status = DataAttribute(id="OBS_STATUS") attr = {"OBS_STATUS": AttributeValue(value_for=obs_status, value="A")} obs = [] for day, value in enumerate([5, 6, 7]): key = key.copy(TIME_PERIOD="2018-01-{:02d}".format(day)) obs.append( Observation(dimension=key, value=value, attached_attribute=attr)) ds = DataSetType() ds.add_obs(obs) # PandasDataSet does not store Observation objects internally, but should # emit them when the .obs property is accessed assert all(a == b for a, b in zip(ds.obs, obs))
def _make_obs(self, key, data): """Create an Observation from tuple *key* and pd.Series *data.""" # Create the Key key = Key( {dim: value for dim, value in zip(self._data.index.names, key)}) attrs = {} # Handle columns of ._data for col, value in data.items(): try: # A tuple column label is ('attr_obs', attr_id) group, attr_id = col except ValueError: # Not a tuple → the 'value' column, handled below continue if group == "attr_obs": # Create a DataAttribute attrs[attr_id] = AttributeValue( value_for=DataAttribute(id=attr_id), value=value) return Observation(dimension=key, value=data["value"], attached_attribute=attrs)
def read_message(self, source, dsd=None): # Initialize message instance msg = DataMessage() if dsd: # pragma: no cover # Store explicit DSD, if any msg.dataflow.structure = dsd # Read JSON source.default_size = -1 tree = json.load(source) # Read the header # TODO handle KeyError here elem = tree["header"] msg.header = Header( id=elem["id"], prepared=elem["prepared"], sender=model.Agency(**elem["sender"]), ) # pre-fetch some structures for efficient use in series and obs structure = tree["structure"] # Read dimensions and values self._dim_level = dict() self._dim_values = dict() for level_name, level in structure["dimensions"].items(): for elem in level: # Create the Dimension d = msg.structure.dimensions.getdefault( id=elem["id"], order=elem.get("keyPosition", -1) ) # Record the level it appears at self._dim_level[d] = level_name # Record values self._dim_values[d] = list() for value in elem.get("values", []): self._dim_values[d].append(KeyValue(id=d.id, value=value["id"])) # Assign an order to an implicit dimension for d in msg.structure.dimensions: if d.order == -1: d.order = len(msg.structure.dimensions) # Determine the dimension at the observation level if all([level == "observation" for level in self._dim_level.values()]): dim_at_obs = AllDimensions else: dim_at_obs = [ dim for dim, level in self._dim_level.items() if level == "observation" ] msg.observation_dimension = dim_at_obs # Read attributes and values self._attr_level = dict() self._attr_values = dict() for level_name, level in structure["attributes"].items(): for attr in level: # Create a DataAttribute in the DSD a = msg.structure.attributes.getdefault( id=attr["id"], concept_identity=Concept(name=attr["name"]) ) # Record the level it appears at self._attr_level[a] = level_name # Record its values self._attr_values[a] = list() for value in attr.get("values", []): self._attr_values[a].append( AttributeValue(value=value["name"], value_for=a) ) self.msg = msg # Make a SeriesKey for Observations in this DataSet ds_key = self._make_key("dataSet") # Read DataSets for ds in tree["dataSets"]: msg.data.append(self.read_dataset(ds, ds_key)) return msg