def test_datastructuredefinition():
dsd = DataStructureDefinition()
# Convenience methods
da = dsd.attributes.getdefault(id="foo")
assert isinstance(da, DataAttribute)
d = dsd.dimensions.getdefault(id="baz", order=-1)
assert isinstance(d, Dimension)
# make_key(GroupKey, ..., extend=True, group_id=None)
gk = dsd.make_key(GroupKey, dict(foo=1, bar=2), extend=True, group_id=None)
# … does not create a GroupDimensionDescriptor (anonymous group)
assert gk.described_by is None
assert len(dsd.group_dimensions) == 0
# But does create the 'bar' dimension
assert "bar" in dsd.dimensions
# make_key(..., group_id=...) creates a GroupDimensionDescriptor
gk = dsd.make_key(GroupKey, dict(foo=1, baz2=4), extend=True, group_id="g1")
assert gk.described_by is dsd.group_dimensions["g1"]
assert len(dsd.group_dimensions) == 1
# …also creates the "baz2" dimension and adds it to the GDD
assert dsd.dimensions.get("baz2") is dsd.group_dimensions["g1"].get("baz2")
# from_keys()
key1 = Key(foo=1, bar=2, baz=3)
key2 = Key(foo=4, bar=5, baz=6)
DataStructureDefinition.from_keys([key1, key2])
def test_dimensiondescriptor():
# from_key()
key1 = Key(foo=1, bar=2, baz=3)
dd = DimensionDescriptor.from_key(key1)
# Key in reverse order
key2 = Key(baz=3, bar=2, foo=1)
assert list(key1.values.keys()) == list(reversed(key2.values.keys()))
key3 = dd.order_key(key2)
assert list(key1.values.keys()) == list(key3.values.keys())
def test_datastructuredefinition():
dsd = DataStructureDefinition()
# Convenience methods
da = dsd.attribute(id='foo')
assert isinstance(da, DataAttribute)
d = dsd.dimension(id='baz', order=-1)
assert isinstance(d, Dimension)
# from_keys()
key1 = Key(foo=1, bar=2, baz=3)
key2 = Key(foo=4, bar=5, baz=6)
dsd.from_keys([key1, key2])
def test_obs(self, msg):
data = msg.data[0]
assert len(data.obs) == 12
series_list = list(data.series)
assert len(series_list) == 4
s3 = series_list[3]
assert isinstance(s3, model.SeriesKey)
# Time series data set → five dimensions are at the SeriesKey level
assert len(s3) == 5
assert s3.CURRENCY == "USD"
# 5 of 7 attributes are at the Observation level
assert len(s3.attrib) == 5
assert s3.attrib.DECIMALS == "4"
obs_list = data.series[s3]
assert len(obs_list) == 3
o0 = obs_list[2]
# One remaining dimension is at the Observation Level
assert len(o0.dimension) == 1
assert o0.dim == Key(TIME_PERIOD="2010-08")
assert o0.value == "1.3898"
# Two remaining attributes are at the Observation level
assert len(o0.attached_attribute)
assert o0.attrib.OBS_STATUS == "A"
def test_structured_obs(self, msg):
data = msg.data[0]
# Expected number of observations and series
assert len(data.obs) == 12
assert len(data.series) == 4
# SeriesKey is accessible by index using DictLike
s3 = list(data.series.keys())[3]
assert isinstance(s3, model.SeriesKey)
# SeriesKey has expected number of dimensions and values
assert len(s3) == 5
assert s3.CURRENCY == "USD"
# SeriesKey has expected number of attributes and values
assert len(s3.attrib) == 5
assert s3.attrib.DECIMALS == "4"
# Series observations can be accessed
obs_list = data.series[s3]
assert len(obs_list) == 3
o0 = obs_list[2]
assert len(o0.dimension) == 1
assert o0.dim == Key(TIME_PERIOD="2010-08")
assert o0.value == "1.3898"
assert o0.attrib.OBS_STATUS == "A"
def parse_obs(self, elem):
values = self._parse(elem)
dd = self._get_current(DimensionDescriptor)
# Attached attributes
aa = values.pop('attributes', {})
if 'obskey' in values:
key = values.pop('obskey')
elif 'obsdimension' in values:
od = values.pop('obsdimension')
assert len(self._obs_dim) == 1
dim = self._obs_dim[0].id
if len(od) == 2:
assert od['id'] == dim, (values, dim)
key = Key(**{dim: od['value']}, described_by=dd)
if len(values):
value = values.pop('obsvalue')
else:
# StructureSpecificData message—all information stored as XML
# attributes of the <Observation>.
attr = copy(elem.attrib)
# Value of the observation
value = attr.pop('OBS_VALUE')
# Dimensions for the key
if self._obs_dim is AllDimensions:
dims = list(attr.keys())
else:
# Use the 'dimension at observation' specified for the message
dims = map(lambda d: d.id, self._obs_dim)
key = Key(**{d: attr.pop(d) for d in dims}, described_by=dd)
# Remaining attr members are SDMX DataAttributes
ad = self._get_current(AttributeDescriptor)
for a_id, a_value in attr.items():
aa[a_id] = AttributeValue(value=a_value,
value_for=ad.get(a_id))
assert len(values) == 0, values
return Observation(dimension=key, value=value, attached_attribute=aa)
def test_key():
# Construct with a dict
k1 = Key({'foo': 1, 'bar': 2})
# Construct with kwargs
k2 = Key(foo=1, bar=2)
# Results are __eq__ each other
assert k1 == k2
# Doing both is an error
with raises(ValueError):
Key({'foo': 1}, bar=2)
# __len__
assert len(k1) == 2
# __contains__: symmetrical if keys are identical
assert k1 in k2
assert k2 in k1
assert Key(foo=1) in k1
assert k1 not in Key(foo=1)
# Set and get using item convenience
k1['baz'] = 3 # bare value is converted to a KeyValue
assert k1['foo'] == 1
# __str__
assert str(k1) == '(foo=1, bar=2, baz=3)'
# copying: returns a new object equal to the old one
k2 = k1.copy()
assert id(k1) != id(k2) and k1 == k2
# copy with changes
k2 = Key(foo=1, bar=2).copy(baz=3)
assert id(k1) != id(k2) and k1 == k2
# __add__: Key with something else
with raises(NotImplementedError):
k1 + 4
# Two Keys
k2 = Key(foo=1) + Key(bar=2)
assert k2 == k1
# __radd__: adding a Key to None produces a Key
assert None + k1 == k1
# anything else is an error
with raises(NotImplementedError):
4 + k1
# get_values(): preserve ordering
assert k1.get_values() == (1, 2, 3)
def parse_obs(self, elem):
# TODO handle key-values as attribs
values = self._parse(elem)
if len(values):
key = (values['obskey']
if 'obskey' in values else values['obsdimension'])
if 'obsdimension' in values:
new_key = Key()
new_key[key.id] = key
key = new_key
obs = Observation(dimension=key,
value=values['obsvalue'],
attached_attribute=values.get('attributes', {}))
else:
# StructureSpecificData message
attr = copy(elem.attrib)
# Value of the observation
value = attr.pop('OBS_VALUE')
# Dimensions for the key
if self._obs_dim is AllDimensions:
dims = list(attr.keys())
else:
# Use the 'dimension at observation' specified for the message
dims = map(lambda d: d.id, self._obs_dim)
# Create the observation, consuming attr for the key
obs = Observation(dimension=Key(**{d: attr.pop(d)
for d in dims}),
value=value)
# Remaining attr members are SDMX DataAttributes
for id, value in attr.items():
da = DataAttribute(id=id)
av = AttributeValue(value_for=da, value=value)
obs.attached_attribute[da.id] = av
return obs
def test_flat():
# Create a bare Message
msg = DataMessage()
# Recreate the content from exr-flat.json
header = Header(
id="62b5f19d-f1c9-495d-8446-a3661ed24753",
prepared="2012-11-29T08:40:26Z",
sender=model.Agency(id="ECB"),
)
msg.header = header
ds = DataSet()
# Create a Key and attributes
key = Key(
FREQ="D",
CURRENCY="NZD",
CURRENCY_DENOM="EUR",
EXR_TYPE="SP00",
EXR_SUFFIX="A",
TIME_PERIOD="2013-01-18",
)
obs_status = DataAttribute(id="OBS_STATUS")
attr = {"OBS_STATUS": AttributeValue(value_for=obs_status, value="A")}
ds.obs.append(
Observation(dimension=key, value=1.5931, attached_attribute=attr))
key = key.copy(TIME_PERIOD="2013-01-21")
ds.obs.append(
Observation(dimension=key, value=1.5925, attached_attribute=attr))
key = key.copy(CURRENCY="RUB", TIME_PERIOD="2013-01-18")
ds.obs.append(
Observation(dimension=key, value=40.3426, attached_attribute=attr))
key = key.copy(TIME_PERIOD="2013-01-21")
ds.obs.append(
Observation(dimension=key, value=40.3000, attached_attribute=attr))
msg.data.append(ds)
# Write to pd.Dataframe
df1 = pandasdmx.to_pandas(msg)
with specimen("flat.json") as f:
ref = pandasdmx.read_sdmx(f)
df2 = pandasdmx.to_pandas(ref)
assert_pd_equal(df1, df2)
def test_flat():
# Create a bare Message
msg = DataMessage()
# Recreate the content from exr-flat.json
header = Header(
id='62b5f19d-f1c9-495d-8446-a3661ed24753',
prepared='2012-11-29T08:40:26Z',
sender='ECB',
)
msg.header = header
ds = DataSet()
# Create a Key and attributes
key = Key(FREQ='D',
CURRENCY='NZD',
CURRENCY_DENOM='EUR',
EXR_TYPE='SP00',
EXR_SUFFIX='A',
TIME_PERIOD='2013-01-18')
obs_status = DataAttribute(id='OBS_STATUS')
attr = {'OBS_STATUS': AttributeValue(value_for=obs_status, value='A')}
ds.obs.append(
Observation(dimension=key, value=1.5931, attached_attribute=attr))
key = key.copy(TIME_PERIOD='2013-01-21')
ds.obs.append(
Observation(dimension=key, value=1.5925, attached_attribute=attr))
key = key.copy(CURRENCY='RUB', TIME_PERIOD='2013-01-18')
ds.obs.append(
Observation(dimension=key, value=40.3426, attached_attribute=attr))
key = key.copy(TIME_PERIOD='2013-01-21')
ds.obs.append(
Observation(dimension=key, value=40.3000, attached_attribute=attr))
msg.data.append(ds)
# Write to pd.Dataframe
df1 = sdmx.to_pandas(msg)
with specimen('flat.json') as f:
ref = sdmx.read_sdmx(f)
df2 = sdmx.to_pandas(ref)
assert_pd_equal(df1, df2)
def parse_obs(self, elem):
values = self._parse(elem)
dsd = self._get_current(DataStructureDefinition)
# Attached attributes
aa = values.pop('attributes', {})
if 'obskey' in values:
key = values.pop('obskey')
elif 'obsdimension' in values:
od = values.pop('obsdimension')
dim = self._obs_dim.id
if len(od) == 2:
assert od['id'] == dim, (values, dim)
key = Key(**{dim: od['value']}, dsd=dsd)
if len(values):
value = values.pop('obsvalue')
else:
# StructureSpecificData message—all information stored as XML
# attributes of the <Observation>.
attr = copy(elem.attrib)
# Value of the observation
value = attr.pop('OBS_VALUE')
# Use the DSD to separate dimensions and attributes
key = Key(**attr, dsd=dsd)
# Remove attributes from the Key to be attached to the Observation
aa.update(key.attrib)
key.attrib = {}
assert len(values) == 0, values
return Observation(dimension=key, value=value, attached_attribute=aa)
def test_add_obs(DataSetType):
# Create a Key and Attributes
key = Key(CURRENCY="NZD", CURRENCY_DENOM="EUR", TIME_PERIOD="2018-01-01")
obs_status = DataAttribute(id="OBS_STATUS")
attr = {"OBS_STATUS": AttributeValue(value_for=obs_status, value="A")}
obs = []
for day, value in enumerate([5, 6, 7]):
key = key.copy(TIME_PERIOD="2018-01-{:02d}".format(day))
obs.append(Observation(dimension=key, value=value, attached_attribute=attr))
ds = DataSetType()
ds.add_obs(obs)
# PandasDataSet does not store Observation objects internally, but should
# emit them when the .obs property is accessed
assert all(a == b for a, b in zip(ds.obs, obs))
def _make_obs(self, key, data):
"""Create an Observation from tuple *key* and pd.Series *data."""
# Create the Key
key = Key(
{dim: value
for dim, value in zip(self._data.index.names, key)})
attrs = {}
# Handle columns of ._data
for col, value in data.items():
try:
# A tuple column label is ('attr_obs', attr_id)
group, attr_id = col
except ValueError:
# Not a tuple → the 'value' column, handled below
continue
if group == 'attr_obs':
# Create a DataAttribute
attrs[attr_id] = AttributeValue(
value_for=DataAttribute(id=attr_id), value=value)
return Observation(dimension=key,
value=data['value'],
attached_attribute=attrs)