def test_maintainable():
urn = "urn:sdmx:org.sdmx.infomodel.conceptscheme.ConceptScheme=IT1:VARIAB_ALL(9.6)"
ma = model.MaintainableArtefact(id="VARIAB_ALL", urn=urn)
# Version is parsed from URN
assert ma.version == "9.6"
# Mismatch raises an exception
with pytest.raises(ValueError, match="Version 9.7 does not match URN"):
model.MaintainableArtefact(version="9.7", urn=urn)
# Maintainer is parsed from URN
assert ma.maintainer == model.Agency(id="IT1")
# Mismatch raises an exception
with pytest.raises(ValueError, match="Maintainer FOO does not match URN"):
model.MaintainableArtefact(maintainer=model.Agency(id="FOO"), urn=urn)
def test_flat():
# Create a bare Message
msg = DataMessage()
# Recreate the content from exr-flat.json
header = Header(
id="62b5f19d-f1c9-495d-8446-a3661ed24753",
prepared="2012-11-29T08:40:26Z",
sender=model.Agency(id="ECB"),
)
msg.header = header
ds = DataSet()
# Create a Key and attributes
key = Key(
FREQ="D",
CURRENCY="NZD",
CURRENCY_DENOM="EUR",
EXR_TYPE="SP00",
EXR_SUFFIX="A",
TIME_PERIOD="2013-01-18",
)
obs_status = DataAttribute(id="OBS_STATUS")
attr = {"OBS_STATUS": AttributeValue(value_for=obs_status, value="A")}
ds.obs.append(
Observation(dimension=key, value=1.5931, attached_attribute=attr))
key = key.copy(TIME_PERIOD="2013-01-21")
ds.obs.append(
Observation(dimension=key, value=1.5925, attached_attribute=attr))
key = key.copy(CURRENCY="RUB", TIME_PERIOD="2013-01-18")
ds.obs.append(
Observation(dimension=key, value=40.3426, attached_attribute=attr))
key = key.copy(TIME_PERIOD="2013-01-21")
ds.obs.append(
Observation(dimension=key, value=40.3000, attached_attribute=attr))
msg.data.append(ds)
# Write to pd.Dataframe
df1 = sdmx.to_pandas(msg)
with specimen("flat.json") as f:
ref = sdmx.read_sdmx(f)
df2 = sdmx.to_pandas(ref)
assert_pd_equal(df1, df2)
def __init__(self, elem, cls_hint=None):
parent_tag = elem.tag
try:
# Use the first child
elem = elem[0]
except IndexError:
raise NotReference
# Extract information from the XML element
if elem.tag == "Ref":
# Element attributes give target_id, id, and version
target_id = elem.attrib["id"]
agency_id = elem.attrib.get("agencyID", None)
id = elem.attrib.get("maintainableParentID", target_id)
version = elem.attrib.get("maintainableParentVersion",
None) or elem.attrib.get(
"version", None)
# Attributes of the element itself, if any
args = (elem.attrib.get("class",
None), elem.attrib.get("package", None))
elif elem.tag == "URN":
match = sdmx.urn.match(elem.text)
# If the URN doesn't specify an item ID, it is probably a reference to a
# MaintainableArtefact, so target_id and id are the same
target_id = match["item_id"] or match["id"]
agency_id = match["agency"]
id = match["id"]
version = match["version"]
args = (match["class"], match["package"])
else:
raise NotReference
# Find the target class
target_cls = model.get_class(*args)
if target_cls is None:
# Try the parent tag name
target_cls = class_for_tag(parent_tag)
if cls_hint and (target_cls is None
or issubclass(cls_hint, target_cls)):
# Hinted class is more specific than target_cls, or failed to find a target
# class above
target_cls = cls_hint
self.maintainable = issubclass(target_cls, model.MaintainableArtefact)
if self.maintainable:
# MaintainableArtefact is the same as the target
cls, id = target_cls, target_id
else:
# Get the class for the parent MaintainableArtefact
cls = model.parent_class(target_cls)
# Store
self.cls = cls
self.agency = model.Agency(id=agency_id) if agency_id else _NO_AGENCY
self.id = id
self.version = version
self.target_cls = target_cls
self.target_id = target_id
return decorator
def to_tags(*args):
return chain(*[[qname(tag) for tag in arg.split()] for arg in args])
PARSE.update({k: None for k in product(to_tags(SKIP), ["start", "end"])})
class NotReference(Exception):
pass
_NO_AGENCY = model.Agency()
class Reference:
"""Temporary class for references.
- `cls`, `id`, `version`, and `agency_id` are always for a MaintainableArtefact.
- If the reference target is a MaintainableArtefact (`maintainable` is True),
`target_cls` and `target_id` are identical to `cls` and `id`, respectively.
- If the target is not maintainable, `target_cls` and `target_id` describe it.
`cls_hint` is an optional hint for when the object is instantiated, i.e. a more
specific override for `cls`/`target_cls`.
"""
def __init__(self, elem, cls_hint=None):
parent_tag = elem.tag
def read_message(self, source, dsd=None):
# Initialize message instance
msg = DataMessage()
if dsd: # pragma: no cover
# Store explicit DSD, if any
msg.dataflow.structure = dsd
# Read JSON
source.default_size = -1
tree = json.load(source)
# Read the header
# TODO handle KeyError here
elem = tree["header"]
msg.header = Header(
id=elem["id"],
prepared=elem["prepared"],
sender=model.Agency(**elem["sender"]),
)
# pre-fetch some structures for efficient use in series and obs
structure = tree["structure"]
# Read dimensions and values
self._dim_level = dict()
self._dim_values = dict()
for level_name, level in structure["dimensions"].items():
for elem in level:
# Create the Dimension
d = msg.structure.dimensions.getdefault(
id=elem["id"], order=elem.get("keyPosition", -1)
)
# Record the level it appears at
self._dim_level[d] = level_name
# Record values
self._dim_values[d] = list()
for value in elem.get("values", []):
self._dim_values[d].append(KeyValue(id=d.id, value=value["id"]))
# Assign an order to an implicit dimension
for d in msg.structure.dimensions:
if d.order == -1:
d.order = len(msg.structure.dimensions)
# Determine the dimension at the observation level
if all([level == "observation" for level in self._dim_level.values()]):
dim_at_obs = AllDimensions
else:
dim_at_obs = [
dim for dim, level in self._dim_level.items() if level == "observation"
]
msg.observation_dimension = dim_at_obs
# Read attributes and values
self._attr_level = dict()
self._attr_values = dict()
for level_name, level in structure["attributes"].items():
for attr in level:
# Create a DataAttribute in the DSD
a = msg.structure.attributes.getdefault(
id=attr["id"], concept_identity=Concept(name=attr["name"])
)
# Record the level it appears at
self._attr_level[a] = level_name
# Record its values
self._attr_values[a] = list()
for value in attr.get("values", []):
self._attr_values[a].append(
AttributeValue(value=value["name"], value_for=a)
)
self.msg = msg
# Make a SeriesKey for Observations in this DataSet
ds_key = self._make_key("dataSet")
# Read DataSets
for ds in tree["dataSets"]:
msg.data.append(self.read_dataset(ds, ds_key))
return msg
def maintainable(self, cls, elem, **kwargs):
"""Create or retrieve a MaintainableArtefact of `cls` from `elem` and `kwargs`.
Following the SDMX-IM class hierachy, :meth:`maintainable` calls
:meth:`nameable`, which in turn calls :meth:`identifiable`, etc. (Since no
concrete class is versionable but not maintainable, no separate method is
created, for better performance). For all of these methods:
- Already-parsed items are removed from the stack only if `elem` is not
:obj:`None`.
- `kwargs` (e.g. 'id') take precedence over any values retrieved from
attributes of `elem`.
If `elem` is None, :meth:`maintainable` returns a MaintainableArtefact with
the is_external_reference attribute set to :obj:`True`. Subsequent calls with
the same object ID will return references to the same object.
"""
kwargs.setdefault("is_external_reference", elem is None)
setdefault_attrib(kwargs, elem, "isExternalReference", "isFinal", "version")
kwargs["is_final"] = kwargs.get("is_final", None) == "true"
# Create a candidate object
obj = self.nameable(cls, elem, **kwargs)
try:
# Retrieve the Agency.id for obj.maintainer
maint = self.get_single(model.Agency, elem.attrib["agencyID"])
except (AttributeError, KeyError):
pass
else:
# Elem contains a maintainer ID
if maint is None:
# …but it did not correspond to an existing object; create one
maint = model.Agency(id=elem.attrib["agencyID"])
self.push(maint)
# This object is never collected; ignore it at end of parsing
self.ignore.add(id(maint))
obj.maintainer = maint
# Maybe retrieve an existing object of the same class and ID
existing = self.get_single(cls, obj.id, strict=True)
if existing and (
existing.compare(obj, strict=True) or existing.urn == sdmx.urn.make(obj)
):
if elem is not None:
# Previously an external reference, now concrete
existing.is_external_reference = False
# Update `existing` from `obj` to preserve references
# If `existing` was a forward reference <Ref/>, its URN was not stored.
for attr in list(kwargs.keys()) + ["urn"]:
# log.info(
# f"Updating {attr} {getattr(existing, attr)} "
# f"{getattr(obj, attr)}"
# )
setattr(existing, attr, getattr(obj, attr))
# Discard the candidate
obj = existing
elif obj.is_external_reference:
# Push a new external reference onto the stack to be located by next calls
self.push(obj)
return obj