def test_flat():
# Create a bare Message
msg = DataMessage()
# Recreate the content from exr-flat.json
header = Header(
id="62b5f19d-f1c9-495d-8446-a3661ed24753",
prepared="2012-11-29T08:40:26Z",
sender=model.Agency(id="ECB"),
)
msg.header = header
ds = DataSet()
# Create a Key and attributes
key = Key(
FREQ="D",
CURRENCY="NZD",
CURRENCY_DENOM="EUR",
EXR_TYPE="SP00",
EXR_SUFFIX="A",
TIME_PERIOD="2013-01-18",
)
obs_status = DataAttribute(id="OBS_STATUS")
attr = {"OBS_STATUS": AttributeValue(value_for=obs_status, value="A")}
ds.obs.append(
Observation(dimension=key, value=1.5931, attached_attribute=attr))
key = key.copy(TIME_PERIOD="2013-01-21")
ds.obs.append(
Observation(dimension=key, value=1.5925, attached_attribute=attr))
key = key.copy(CURRENCY="RUB", TIME_PERIOD="2013-01-18")
ds.obs.append(
Observation(dimension=key, value=40.3426, attached_attribute=attr))
key = key.copy(TIME_PERIOD="2013-01-21")
ds.obs.append(
Observation(dimension=key, value=40.3000, attached_attribute=attr))
msg.data.append(ds)
# Write to pd.Dataframe
df1 = pandasdmx.to_pandas(msg)
with specimen("flat.json") as f:
ref = pandasdmx.read_sdmx(f)
df2 = pandasdmx.to_pandas(ref)
assert_pd_equal(df1, df2)
def test_flat():
# Create a bare Message
msg = DataMessage()
# Recreate the content from exr-flat.json
header = Header(
id='62b5f19d-f1c9-495d-8446-a3661ed24753',
prepared='2012-11-29T08:40:26Z',
sender='ECB',
)
msg.header = header
ds = DataSet()
# Create a Key and attributes
key = Key(FREQ='D',
CURRENCY='NZD',
CURRENCY_DENOM='EUR',
EXR_TYPE='SP00',
EXR_SUFFIX='A',
TIME_PERIOD='2013-01-18')
obs_status = DataAttribute(id='OBS_STATUS')
attr = {'OBS_STATUS': AttributeValue(value_for=obs_status, value='A')}
ds.obs.append(
Observation(dimension=key, value=1.5931, attached_attribute=attr))
key = key.copy(TIME_PERIOD='2013-01-21')
ds.obs.append(
Observation(dimension=key, value=1.5925, attached_attribute=attr))
key = key.copy(CURRENCY='RUB', TIME_PERIOD='2013-01-18')
ds.obs.append(
Observation(dimension=key, value=40.3426, attached_attribute=attr))
key = key.copy(TIME_PERIOD='2013-01-21')
ds.obs.append(
Observation(dimension=key, value=40.3000, attached_attribute=attr))
msg.data.append(ds)
# Write to pd.Dataframe
df1 = sdmx.to_pandas(msg)
with specimen('flat.json') as f:
ref = sdmx.read_sdmx(f)
df2 = sdmx.to_pandas(ref)
assert_pd_equal(df1, df2)
def read_obs(self, root, series_key=None, base_key=None):
for key, elem in root.get('observations', {}).items():
value = elem.pop(0) if len(elem) else None
o = Observation(
series_key=series_key,
dimension=self._make_key('observation', key, base=base_key),
value=value,
attached_attribute=self._make_attrs('observation', elem))
yield o
def test_observation():
obs = Observation()
# Set by item name
obs.attached_attribute['TIME_PERIOD'] = 3
# NB the following does not work; see Observation.attrib()
# obs.attrib['TIME_PERIOD'] = 3
obs.attached_attribute['CURRENCY'] = 'USD'
# Access by attribute name
assert obs.attrib.TIME_PERIOD == 3
assert obs.attrib.CURRENCY == 'USD'
# Access by item index
assert obs.attrib[1] == 'USD'
# Add attributes
obs.attached_attribute['FOO'] = '1'
obs.attached_attribute['BAR'] = '2'
assert obs.attrib.FOO == '1' and obs.attrib['BAR'] == '2'
# Using classes
da = DataAttribute(id='FOO')
av = AttributeValue(value_for=da, value='baz')
obs.attached_attribute[da.id] = av
assert obs.attrib[da.id] == 'baz'
def test_observation():
obs = Observation()
# Set by item name
obs.attached_attribute["TIME_PERIOD"] = 3
# NB the following does not work; see Observation.attrib()
# obs.attrib['TIME_PERIOD'] = 3
obs.attached_attribute["CURRENCY"] = "USD"
# Access by attribute name
assert obs.attrib.TIME_PERIOD == 3
assert obs.attrib.CURRENCY == "USD"
# Access by item index
assert obs.attrib[1] == "USD"
# Add attributes
obs.attached_attribute["FOO"] = "1"
obs.attached_attribute["BAR"] = "2"
assert obs.attrib.FOO == "1" and obs.attrib["BAR"] == "2"
# Using classes
da = DataAttribute(id="FOO")
av = AttributeValue(value_for=da, value="baz")
obs.attached_attribute[da.id] = av
assert obs.attrib[da.id] == "baz"
def parse_obs(self, elem):
# TODO handle key-values as attribs
values = self._parse(elem)
if len(values):
key = (values['obskey']
if 'obskey' in values else values['obsdimension'])
if 'obsdimension' in values:
new_key = Key()
new_key[key.id] = key
key = new_key
obs = Observation(dimension=key,
value=values['obsvalue'],
attached_attribute=values.get('attributes', {}))
else:
# StructureSpecificData message
attr = copy(elem.attrib)
# Value of the observation
value = attr.pop('OBS_VALUE')
# Dimensions for the key
if self._obs_dim is AllDimensions:
dims = list(attr.keys())
else:
# Use the 'dimension at observation' specified for the message
dims = map(lambda d: d.id, self._obs_dim)
# Create the observation, consuming attr for the key
obs = Observation(dimension=Key(**{d: attr.pop(d)
for d in dims}),
value=value)
# Remaining attr members are SDMX DataAttributes
for id, value in attr.items():
da = DataAttribute(id=id)
av = AttributeValue(value_for=da, value=value)
obs.attached_attribute[da.id] = av
return obs
def test_add_obs(DataSetType):
# Create a Key and Attributes
key = Key(CURRENCY="NZD", CURRENCY_DENOM="EUR", TIME_PERIOD="2018-01-01")
obs_status = DataAttribute(id="OBS_STATUS")
attr = {"OBS_STATUS": AttributeValue(value_for=obs_status, value="A")}
obs = []
for day, value in enumerate([5, 6, 7]):
key = key.copy(TIME_PERIOD="2018-01-{:02d}".format(day))
obs.append(Observation(dimension=key, value=value, attached_attribute=attr))
ds = DataSetType()
ds.add_obs(obs)
# PandasDataSet does not store Observation objects internally, but should
# emit them when the .obs property is accessed
assert all(a == b for a, b in zip(ds.obs, obs))
def parse_obs(self, elem):
values = self._parse(elem)
dd = self._get_current(DimensionDescriptor)
# Attached attributes
aa = values.pop('attributes', {})
if 'obskey' in values:
key = values.pop('obskey')
elif 'obsdimension' in values:
od = values.pop('obsdimension')
assert len(self._obs_dim) == 1
dim = self._obs_dim[0].id
if len(od) == 2:
assert od['id'] == dim, (values, dim)
key = Key(**{dim: od['value']}, described_by=dd)
if len(values):
value = values.pop('obsvalue')
else:
# StructureSpecificData message—all information stored as XML
# attributes of the <Observation>.
attr = copy(elem.attrib)
# Value of the observation
value = attr.pop('OBS_VALUE')
# Dimensions for the key
if self._obs_dim is AllDimensions:
dims = list(attr.keys())
else:
# Use the 'dimension at observation' specified for the message
dims = map(lambda d: d.id, self._obs_dim)
key = Key(**{d: attr.pop(d) for d in dims}, described_by=dd)
# Remaining attr members are SDMX DataAttributes
ad = self._get_current(AttributeDescriptor)
for a_id, a_value in attr.items():
aa[a_id] = AttributeValue(value=a_value,
value_for=ad.get(a_id))
assert len(values) == 0, values
return Observation(dimension=key, value=value, attached_attribute=aa)
def _make_obs(self, key, data):
"""Create an Observation from tuple *key* and pd.Series *data."""
# Create the Key
key = Key(
{dim: value
for dim, value in zip(self._data.index.names, key)})
attrs = {}
# Handle columns of ._data
for col, value in data.items():
try:
# A tuple column label is ('attr_obs', attr_id)
group, attr_id = col
except ValueError:
# Not a tuple → the 'value' column, handled below
continue
if group == 'attr_obs':
# Create a DataAttribute
attrs[attr_id] = AttributeValue(
value_for=DataAttribute(id=attr_id), value=value)
return Observation(dimension=key,
value=data['value'],
attached_attribute=attrs)
def parse_obs(self, elem):
values = self._parse(elem)
dsd = self._get_current(DataStructureDefinition)
# Attached attributes
aa = values.pop('attributes', {})
if 'obskey' in values:
key = values.pop('obskey')
elif 'obsdimension' in values:
od = values.pop('obsdimension')
dim = self._obs_dim.id
if len(od) == 2:
assert od['id'] == dim, (values, dim)
key = Key(**{dim: od['value']}, dsd=dsd)
if len(values):
value = values.pop('obsvalue')
else:
# StructureSpecificData message—all information stored as XML
# attributes of the <Observation>.
attr = copy(elem.attrib)
# Value of the observation
value = attr.pop('OBS_VALUE')
# Use the DSD to separate dimensions and attributes
key = Key(**attr, dsd=dsd)
# Remove attributes from the Key to be attached to the Observation
aa.update(key.attrib)
key.attrib = {}
assert len(values) == 0, values
return Observation(dimension=key, value=value, attached_attribute=aa)
