def test_refcount():
# Component (subclasses) created outside of a DataStructureDefintion
da1 = DataAttribute(id='foo')
da2 = DataAttribute(id='foo')
assert id(da1) != id(da2)
assert da1 is not da2
# Retrieving attributes from a DataStructureDefintion results in references
# to the same object
dsd = DataStructureDefinition()
da3 = dsd.attribute('foo')
da4 = dsd.attribute('foo')
assert id(da3) == id(da4)
assert da3 is da4
assert len(dsd.attributes) == 1
# Creating an AttributeValue referencing a DataAttribute outside a DSD
av1 = AttributeValue(value='baz', value_for=DataAttribute(id='foo'))
assert not any([av1.value_for is da for da in (da1, da2, da3)])
# Same, using a DSD
av2 = AttributeValue(value='baz', value_for='foo', dsd=dsd)
assert av2.value_for is da3
def parse_attributes(self, elem):
result = {}
for e in elem.iterchildren():
da = DataAttribute(id=e.attrib['id'])
av = AttributeValue(value_for=da, value=e.attrib['value'])
result[da.id] = av
return result
def test_observation():
obs = Observation()
# Set by item name
obs.attached_attribute['TIME_PERIOD'] = 3
# NB the following does not work; see Observation.attrib()
# obs.attrib['TIME_PERIOD'] = 3
obs.attached_attribute['CURRENCY'] = 'USD'
# Access by attribute name
assert obs.attrib.TIME_PERIOD == 3
assert obs.attrib.CURRENCY == 'USD'
# Access by item index
assert obs.attrib[1] == 'USD'
# Add attributes
obs.attached_attribute['FOO'] = '1'
obs.attached_attribute['BAR'] = '2'
assert obs.attrib.FOO == '1' and obs.attrib['BAR'] == '2'
# Using classes
da = DataAttribute(id='FOO')
av = AttributeValue(value_for=da, value='baz')
obs.attached_attribute[da.id] = av
assert obs.attrib[da.id] == 'baz'
def test_observation():
obs = Observation()
# Set by item name
obs.attached_attribute["TIME_PERIOD"] = 3
# NB the following does not work; see Observation.attrib()
# obs.attrib['TIME_PERIOD'] = 3
obs.attached_attribute["CURRENCY"] = "USD"
# Access by attribute name
assert obs.attrib.TIME_PERIOD == 3
assert obs.attrib.CURRENCY == "USD"
# Access by item index
assert obs.attrib[1] == "USD"
# Add attributes
obs.attached_attribute["FOO"] = "1"
obs.attached_attribute["BAR"] = "2"
assert obs.attrib.FOO == "1" and obs.attrib["BAR"] == "2"
# Using classes
da = DataAttribute(id="FOO")
av = AttributeValue(value_for=da, value="baz")
obs.attached_attribute[da.id] = av
assert obs.attrib[da.id] == "baz"
def parse_attributes(self, elem):
result = {}
ad = self._get_current(AttributeDescriptor)
for e in elem.iterchildren():
da = ad.get(e.attrib['id'])
av = AttributeValue(value=e.attrib['value'], value_for=da)
result[da.id] = av
return result
def test_flat():
# Create a bare Message
msg = DataMessage()
# Recreate the content from exr-flat.json
header = Header(
id="62b5f19d-f1c9-495d-8446-a3661ed24753",
prepared="2012-11-29T08:40:26Z",
sender=model.Agency(id="ECB"),
)
msg.header = header
ds = DataSet()
# Create a Key and attributes
key = Key(
FREQ="D",
CURRENCY="NZD",
CURRENCY_DENOM="EUR",
EXR_TYPE="SP00",
EXR_SUFFIX="A",
TIME_PERIOD="2013-01-18",
)
obs_status = DataAttribute(id="OBS_STATUS")
attr = {"OBS_STATUS": AttributeValue(value_for=obs_status, value="A")}
ds.obs.append(
Observation(dimension=key, value=1.5931, attached_attribute=attr))
key = key.copy(TIME_PERIOD="2013-01-21")
ds.obs.append(
Observation(dimension=key, value=1.5925, attached_attribute=attr))
key = key.copy(CURRENCY="RUB", TIME_PERIOD="2013-01-18")
ds.obs.append(
Observation(dimension=key, value=40.3426, attached_attribute=attr))
key = key.copy(TIME_PERIOD="2013-01-21")
ds.obs.append(
Observation(dimension=key, value=40.3000, attached_attribute=attr))
msg.data.append(ds)
# Write to pd.Dataframe
df1 = pandasdmx.to_pandas(msg)
with specimen("flat.json") as f:
ref = pandasdmx.read_sdmx(f)
df2 = pandasdmx.to_pandas(ref)
assert_pd_equal(df1, df2)
def test_flat():
# Create a bare Message
msg = DataMessage()
# Recreate the content from exr-flat.json
header = Header(
id='62b5f19d-f1c9-495d-8446-a3661ed24753',
prepared='2012-11-29T08:40:26Z',
sender='ECB',
)
msg.header = header
ds = DataSet()
# Create a Key and attributes
key = Key(FREQ='D',
CURRENCY='NZD',
CURRENCY_DENOM='EUR',
EXR_TYPE='SP00',
EXR_SUFFIX='A',
TIME_PERIOD='2013-01-18')
obs_status = DataAttribute(id='OBS_STATUS')
attr = {'OBS_STATUS': AttributeValue(value_for=obs_status, value='A')}
ds.obs.append(
Observation(dimension=key, value=1.5931, attached_attribute=attr))
key = key.copy(TIME_PERIOD='2013-01-21')
ds.obs.append(
Observation(dimension=key, value=1.5925, attached_attribute=attr))
key = key.copy(CURRENCY='RUB', TIME_PERIOD='2013-01-18')
ds.obs.append(
Observation(dimension=key, value=40.3426, attached_attribute=attr))
key = key.copy(TIME_PERIOD='2013-01-21')
ds.obs.append(
Observation(dimension=key, value=40.3000, attached_attribute=attr))
msg.data.append(ds)
# Write to pd.Dataframe
df1 = sdmx.to_pandas(msg)
with specimen('flat.json') as f:
ref = sdmx.read_sdmx(f)
df2 = sdmx.to_pandas(ref)
assert_pd_equal(df1, df2)
def test_add_obs(DataSetType):
# Create a Key and Attributes
key = Key(CURRENCY="NZD", CURRENCY_DENOM="EUR", TIME_PERIOD="2018-01-01")
obs_status = DataAttribute(id="OBS_STATUS")
attr = {"OBS_STATUS": AttributeValue(value_for=obs_status, value="A")}
obs = []
for day, value in enumerate([5, 6, 7]):
key = key.copy(TIME_PERIOD="2018-01-{:02d}".format(day))
obs.append(Observation(dimension=key, value=value, attached_attribute=attr))
ds = DataSetType()
ds.add_obs(obs)
# PandasDataSet does not store Observation objects internally, but should
# emit them when the .obs property is accessed
assert all(a == b for a, b in zip(ds.obs, obs))
def parse_obs(self, elem):
values = self._parse(elem)
dd = self._get_current(DimensionDescriptor)
# Attached attributes
aa = values.pop('attributes', {})
if 'obskey' in values:
key = values.pop('obskey')
elif 'obsdimension' in values:
od = values.pop('obsdimension')
assert len(self._obs_dim) == 1
dim = self._obs_dim[0].id
if len(od) == 2:
assert od['id'] == dim, (values, dim)
key = Key(**{dim: od['value']}, described_by=dd)
if len(values):
value = values.pop('obsvalue')
else:
# StructureSpecificData message—all information stored as XML
# attributes of the <Observation>.
attr = copy(elem.attrib)
# Value of the observation
value = attr.pop('OBS_VALUE')
# Dimensions for the key
if self._obs_dim is AllDimensions:
dims = list(attr.keys())
else:
# Use the 'dimension at observation' specified for the message
dims = map(lambda d: d.id, self._obs_dim)
key = Key(**{d: attr.pop(d) for d in dims}, described_by=dd)
# Remaining attr members are SDMX DataAttributes
ad = self._get_current(AttributeDescriptor)
for a_id, a_value in attr.items():
aa[a_id] = AttributeValue(value=a_value,
value_for=ad.get(a_id))
assert len(values) == 0, values
return Observation(dimension=key, value=value, attached_attribute=aa)
def parse_group(self, elem):
"""<generic:Group>, <structure:Group>, or <Group>."""
values = self._parse(elem)
# Check which namespace this Group tag is part of
if elem.tag == qname('gen', 'Group'):
# generic → GroupKey in a DataMessage
gk = values.pop('groupkey')
gk.attrib.update(values.pop('attributes', {}))
result = gk
elif elem.tag == qname('str', 'Group'):
# structure → GroupDimensionDescriptor
gdd = GroupDimensionDescriptor(id=elem.attrib['id'],
components=wrap(
values.pop('groupdimension')))
# Early update of the DSD so that later definitions in the DSD can
# reference gdd
dsd = self._get_current(DataStructureDefinition)
dsd.group_dimensions = gdd
result = gdd
else:
# no namespace → GroupKey in a StructureSpecificData message
# commented: destructive
# # Discard XML Schema attribute
# elem.attrib.pop(qname('xsi', 'type'))
# the 'TITLE' XML attribute is an SDMX Attribute
attrib = {}
try:
da = DataAttribute(id='TITLE')
av = AttributeValue(value=elem.attrib['TITLE'], value_for=da)
attrib[da.id] = av
except KeyError:
pass
# Remaining attributes are the KeyValues
result = GroupKey(**elem.attrib, attrib=attrib)
assert len(values) == 0
return result
def parse_obs(self, elem):
# TODO handle key-values as attribs
values = self._parse(elem)
if len(values):
key = (values['obskey']
if 'obskey' in values else values['obsdimension'])
if 'obsdimension' in values:
new_key = Key()
new_key[key.id] = key
key = new_key
obs = Observation(dimension=key,
value=values['obsvalue'],
attached_attribute=values.get('attributes', {}))
else:
# StructureSpecificData message
attr = copy(elem.attrib)
# Value of the observation
value = attr.pop('OBS_VALUE')
# Dimensions for the key
if self._obs_dim is AllDimensions:
dims = list(attr.keys())
else:
# Use the 'dimension at observation' specified for the message
dims = map(lambda d: d.id, self._obs_dim)
# Create the observation, consuming attr for the key
obs = Observation(dimension=Key(**{d: attr.pop(d)
for d in dims}),
value=value)
# Remaining attr members are SDMX DataAttributes
for id, value in attr.items():
da = DataAttribute(id=id)
av = AttributeValue(value_for=da, value=value)
obs.attached_attribute[da.id] = av
return obs
def _make_obs(self, key, data):
"""Create an Observation from tuple *key* and pd.Series *data."""
# Create the Key
key = Key(
{dim: value
for dim, value in zip(self._data.index.names, key)})
attrs = {}
# Handle columns of ._data
for col, value in data.items():
try:
# A tuple column label is ('attr_obs', attr_id)
group, attr_id = col
except ValueError:
# Not a tuple → the 'value' column, handled below
continue
if group == 'attr_obs':
# Create a DataAttribute
attrs[attr_id] = AttributeValue(
value_for=DataAttribute(id=attr_id), value=value)
return Observation(dimension=key,
value=data['value'],
attached_attribute=attrs)
def read_message(self, source):
# Initialize message instance
msg = DataMessage()
# Read JSON
tree = json.load(source)
# Read the header
# FIXME KeyError: 'header'
elem = tree['header']
msg.header = Header(id=elem['id'],
prepared=elem['prepared'],
sender=Item(**elem['sender']))
# pre-fetch some structures for efficient use in series and obs
structure = tree['structure']
# Read dimensions and values
self._dim_level = dict()
self._dim_values = dict()
for level_name, level in structure['dimensions'].items():
for elem in level:
# Create the Dimension
d = msg.structure.dimension(id=elem['id'],
order=elem.get('keyPosition', -1))
# Record the level it appears at
self._dim_level[d] = level_name
# Record values
self._dim_values[d] = list()
for value in elem.get('values', []):
self._dim_values[d].append(
KeyValue(id=d.id, value=value['id']))
# Assign an order to an implicit dimension
for d in msg.structure.dimensions:
if d.order == -1:
d.order = len(msg.structure.dimensions)
# Determine the dimension at the observation level
if all([level == 'observation' for level in self._dim_level.values()]):
dim_at_obs = AllDimensions
else:
dim_at_obs = [
dim for dim, level in self._dim_level.items()
if level == 'observation'
]
msg.observation_dimension = dim_at_obs
# Read attributes and values
self._attr_level = dict()
self._attr_values = dict()
for level_name, level in structure['attributes'].items():
for attr in level:
# Create a DataAttribute in the DSD
a = msg.structure.attribute(
id=attr['id'],
concept_identity=Concept(name=attr['name']),
)
# Record the level it appears at
self._attr_level[a] = level_name
# Record its values
self._attr_values[a] = list()
for value in attr.get('values', []):
self._attr_values[a].append(
AttributeValue(value=value['name'], value_for=a))
self.msg = msg
# Make a SeriesKey for Observations in this DataSet
ds_key = self._make_key('dataSet')
# Read DataSets
for ds in tree['dataSets']:
msg.data.append(self.read_dataset(ds, ds_key))
return msg
def read_message(self, source, dsd=None):
# Initialize message instance
msg = DataMessage()
if dsd: # pragma: no cover
# Store explicit DSD, if any
msg.dataflow.structure = dsd
# Read JSON
source.default_size = -1
tree = json.load(source)
# Read the header
# TODO handle KeyError here
elem = tree["header"]
msg.header = Header(
id=elem["id"],
prepared=elem["prepared"],
sender=model.Agency(**elem["sender"]),
)
# pre-fetch some structures for efficient use in series and obs
structure = tree["structure"]
# Read dimensions and values
self._dim_level = dict()
self._dim_values = dict()
for level_name, level in structure["dimensions"].items():
for elem in level:
# Create the Dimension
d = msg.structure.dimensions.getdefault(
id=elem["id"], order=elem.get("keyPosition", -1)
)
# Record the level it appears at
self._dim_level[d] = level_name
# Record values
self._dim_values[d] = list()
for value in elem.get("values", []):
self._dim_values[d].append(KeyValue(id=d.id, value=value["id"]))
# Assign an order to an implicit dimension
for d in msg.structure.dimensions:
if d.order == -1:
d.order = len(msg.structure.dimensions)
# Determine the dimension at the observation level
if all([level == "observation" for level in self._dim_level.values()]):
dim_at_obs = AllDimensions
else:
dim_at_obs = [
dim for dim, level in self._dim_level.items() if level == "observation"
]
msg.observation_dimension = dim_at_obs
# Read attributes and values
self._attr_level = dict()
self._attr_values = dict()
for level_name, level in structure["attributes"].items():
for attr in level:
# Create a DataAttribute in the DSD
a = msg.structure.attributes.getdefault(
id=attr["id"], concept_identity=Concept(name=attr["name"])
)
# Record the level it appears at
self._attr_level[a] = level_name
# Record its values
self._attr_values[a] = list()
for value in attr.get("values", []):
self._attr_values[a].append(
AttributeValue(value=value["name"], value_for=a)
)
self.msg = msg
# Make a SeriesKey for Observations in this DataSet
ds_key = self._make_key("dataSet")
# Read DataSets
for ds in tree["dataSets"]:
msg.data.append(self.read_dataset(ds, ds_key))
return msg