class TestGenericSeriesData_RateGroup_TS(unittest.TestCase):
def setUp(self):
self.estat = Request()
filepath = os.path.join(
pkg_path, 'data/exr/ecb_exr_rg/generic/ecb_exr_rg_ts.xml')
self.resp = self.estat.get(fromfile=filepath)
def test_groups(self):
data = self.resp.data
self.assertEqual(len(list(data.groups)), 5)
self.assertEqual(len(list(data.series)), 4)
g2 = list(data.groups)[2]
self.assertEqual(g2.key.CURRENCY, 'GBP')
self.assertEqual(
g2.attrib.TITLE, 'ECB reference exchange rate, U.K. Pound sterling /Euro')
# Check group attributes of a series
s = list(data.series)[0]
g_attrib = s.group_attrib
self.assertEqual(len(g_attrib), 5)
self.assertIsInstance(g_attrib, tuple)
self.assertEqual(len(g_attrib), 5)
def test_footer(self):
filepath = os.path.join(
pkg_path, 'data/estat/footer.xml')
resp = self.estat.get(
fromfile=filepath, get_footer_url=None)
f = resp.footer
assert f.code == 413
assert f.severity == 'Infomation'
assert f.text[1].startswith('http')
def get(self, request, *args, **kwargs):
term = request.GET.get('q')
if term is None:
return Response({'error': 'Invalid parameters.'},
status=status.HTTP_400_BAD_REQUEST)
estat = Request("ESTAT")
dataflow = estat.get(resource_type='dataflow')
dataflows = dataflow.msg.dataflows
dsd_id = dataflows[term].structure.id
dsd_resp = estat.get(resource_type='datastructure', resource_id=dsd_id)
dsd = dsd_resp.msg.datastructures[dsd_id]
dimensionsList = list(dsd.dimensions)
dimensionsWithValues = {}
for f in range(0, len(dimensionsList)):
if (dimensionsList[f] != "TIME_PERIOD"):
dimensionsValuesList = list(
dsd.dimensions[dimensionsList[f]].local_repr.enum.values())
valuesList = []
for value in range(0, len(dimensionsValuesList)):
valuesList.append([
dimensionsValuesList[value].id,
dimensionsValuesList[value].name.en
])
dimensionsWithValues.update({dimensionsList[f]: valuesList})
elif (dimensionsList[f] == "TIME_PERIOD"):
print("TIME ", dsd.dimensions[dimensionsList[f]].concept.name)
filters = {"result": dimensionsWithValues}
filtersString = str(filters).replace("{'", '{"')
filtersString = str(filtersString).replace("':", '":')
filtersString = str(filtersString).replace("['", '["')
filtersString = str(filtersString).replace("]'", ']"')
filtersString = str(filtersString).replace("']", '"]')
filtersString = str(filtersString).replace("',", '",')
filtersString = str(filtersString).replace(", '", ', "')
filterJson = json.loads(filtersString)
return Response(filterJson)
def get(self, request, *args, **kwargs):
start = request.GET.get('start')
term = request.GET.get('q')
if start is None:
start = "0"
if term is None:
return Response({'error': 'Invalid parameters.'},
status=status.HTTP_400_BAD_REQUEST)
estat = Request("ESTAT")
dataflows = estat.get(resource_type='dataflow').msg.dataflows
searchResults = dataflows.find(term)
searchResultsKeys = list(dataflows.find(term))
resultList = []
for key in searchResultsKeys:
resultList.append([key, searchResults[key].name["en"]])
results = {"result": resultList}
return Response(results)
class Test_ESTAT_dsd_apro_mk_cola(unittest.TestCase):
def setUp(self):
self.estat = Request('ESTAT')
filepath = os.path.join(test_path, 'data/estat/apro_dsd.xml')
self.resp = self.estat.get(fromfile=filepath)
def test_codelists_keys(self):
self.assertEqual(len(self.resp.codelist), 6)
self.assertIsInstance(self.resp.codelist.CL_GEO, model.Codelist)
def test_codelist_name(self):
self.assertEqual(
self.resp.msg.codelist.CL_GEO.UK.name.en, 'United Kingdom')
assert self.resp.codelist.CL_FREQ.name.en == 'FREQ'
def test_code_cls(self):
self.assertIsInstance(
self.resp.codelist.CL_FREQ.D, model.Code)
def test_writer(self):
df = self.resp.write(rows='codelist')
self.assertEqual(df.shape, (79, 2))
def tearDown(self):
pass
class TestStructSpecSeriesData_RateGroup_TS(unittest.TestCase):
def setUp(self):
self.estat = Request()
filepath = os.path.join(
pkg_path, 'data/exr/ecb_exr_rg/structured/ecb_exr_rg_ts.xml')
dsd_resp = self.estat.datastructure(
fromfile=os.path.join(pkg_path, 'data/exr/ecb_exr_rg/ecb_exr_rg.xml'))
dsd = dsd_resp.datastructure.DataStructure
self.resp = self.estat.get(fromfile=filepath, dsd=dsd)
def test_groups(self):
data = self.resp.data
self.assertEqual(len(list(data.groups)), 5)
self.assertEqual(len(list(data.series)), 4)
g2 = list(data.groups)[2]
self.assertEqual(g2.key.CURRENCY, 'GBP')
self.assertEqual(
g2.attrib.TITLE, 'ECB reference exchange rate, U.K. Pound sterling /Euro')
# Check group attributes of a series
s = list(data.series)[0]
g_attrib = s.group_attrib
self.assertEqual(len(g_attrib), 5)
self.assertIsInstance(g_attrib, tuple)
self.assertEqual(len(g_attrib), 5)
class test_dsd_common(unittest.TestCase):
def setUp(self):
self.estat = Request('ESTAT')
filepath = os.path.join(test_path, 'data/common/common.xml')
self.resp = self.estat.get(fromfile=filepath)
def test_codelists_keys(self):
self.assertEqual(len(self.resp.msg.codelist), 5)
self.assertIsInstance(self.resp.msg.codelist.CL_FREQ, model.Codelist)
def test_codelist_name(self):
self.assertEqual(self.resp.msg.codelist.CL_FREQ.D.name.en, 'Daily')
def test_code_cls(self):
self.assertIsInstance(self.resp.msg.codelist.CL_FREQ.D, model.Code)
def test_annotations(self):
code = self.resp.codelist.CL_FREQ.A
anno_list = list(code.annotations)
self.assertEqual(len(anno_list), 1)
a = anno_list[0]
self.assertIsInstance(a, model.Annotation)
self.assertTrue(a.text.en.startswith('It is'))
self.assertEqual(a.annotationtype, 'NOTE')
def test_doc_index1():
"""First code example in index.rst."""
estat = Request('ESTAT')
flow_response = estat.dataflow('une_rt_a')
with pytest.raises(TypeError):
# This presumes the DataStructureDefinition instance can conduct a
# network request for its own content
structure_response = flow_response.dataflow.une_rt_a.structure(
request=True, target_only=False)
# Same effect
structure_response = estat.get(
'datastructure', flow_response.dataflow.une_rt_a.structure.id)
# Even better: Request.get(…) should examine the class and ID of the object
# structure = estat.get(flow_response.dataflow.une_rt_a.structure)
# Show some codelists
s = sdmx.to_pandas(structure_response)
expected = pd.Series({
'AT': 'Austria',
'BE': 'Belgium',
'BG': 'Bulgaria',
'CH': 'Switzerland',
'CY': 'Cyprus',
}, name='GEO') \
.rename_axis('CL_GEO')
# Codelists are converted to a DictLike
assert isinstance(s.codelist, DictLike)
# Same effect
assert_pd_equal(s.codelist['CL_GEO'].sort_index().head(), expected)
class TestStructSpecSeriesData_RateGroup_TS(unittest.TestCase):
def setUp(self):
self.estat = Request()
filepath = os.path.join(
pkg_path, 'data/exr/ecb_exr_rg/structured/ecb_exr_rg_ts.xml')
dsd_resp = self.estat.datastructure(
fromfile=os.path.join(pkg_path, 'data/exr/ecb_exr_rg/ecb_exr_rg.xml'))
dsd = dsd_resp.datastructure.DataStructure
self.resp = self.estat.get(fromfile=filepath, dsd=dsd)
def test_groups(self):
data = self.resp.data
self.assertEqual(len(list(data.groups)), 5)
self.assertEqual(len(list(data.series)), 4)
g2 = list(data.groups)[2]
self.assertEqual(g2.key.CURRENCY, 'GBP')
self.assertEqual(
g2.attrib.TITLE, 'ECB reference exchange rate, U.K. Pound sterling /Euro')
# Check group attributes of a series
s = list(data.series)[0]
g_attrib = s.group_attrib
self.assertEqual(len(g_attrib), 5)
self.assertIsInstance(g_attrib, tuple)
self.assertEqual(len(g_attrib), 5)
def test_samples_request(self):
"""Test the samples from the SDMXJSON spec."""
req = Request()
for name, data in sample_data.items():
resp = req.get(fromfile=self._filepath(name))
df = resp.write()
assert df.equals(data), \
'\n'.join(map(str, [name, df.index, data.index,
getattr(df, 'columns', ''),
getattr(data, 'columns', '')]))
def test_samples_request(self):
"""Test the samples from the SDMXJSON spec."""
req = Request()
for name, data in sample_data.items():
resp = req.get(fromfile=self._filepath(name))
df = resp.write()
assert df.equals(data), \
'\n'.join(map(str, [name, df.index, data.index,
getattr(df, 'columns', ''),
getattr(data, 'columns', '')]))
def test1():
ecb = Request('ECB')
cat_resp = ecb.get(resource_type='categoryscheme')
cat_msg = cat_resp.msg
print dir(cat_msg)
print str(cat_msg.header)
print str(cat_msg.categoryscheme)
print str(cat_msg.dataflow)
for key, value in cat_msg.dataflow.iteritems():
print "{k}:{v}".format(k=key, v=value)
# resource must be one of ['dataflow', 'datastructure', 'data', 'categoryscheme', 'codelist', 'conceptscheme']
flows = ecb.get(resource_type='dataflow')
print str(flows)
refs = dict(references='all')
#dsd_resp = ecb.get(resource_type = 'datastructure', resource_id = 'EXR_PUB', params = refs)
dsd_id = cat_msg.dataflows.EXR_PUB.structure.id
dsd_id = cat_msg.dataflows.EXR.structure.id
dsd_id = 'ECB_EXR1'
dsd_resp = ecb.get(resource_type='datastructure',
resource_id=dsd_id,
params=refs)
#print str(dsd_resp)
print dir(dsd_resp)
dsd = dsd_resp.msg.datastructures[dsd_id]
print str(dsd)
data_resp = ecb.get(resource_type='data',
resource_id='EXR',
key={'CURRENCY': 'USD+JPY'},
params={'startPeriod': '2014'})
print dir(data_resp)
data = data_resp.msg.data
#print dir(data.series)
series_l = list(data.series)
print series_l
def test_write_source(self):
"""Test the write_source() method."""
req = Request()
for name in sample_data.keys():
orig_fn = self._filepath(name)
temp_fn = self._filepath(name + '-write-source')
# Read the message
resp = req.get(fromfile=orig_fn)
# Write to a temporary JSON file
resp.write_source(temp_fn)
# Read the two files and compare JSON (ignores ordering)
with open(orig_fn) as orig, open(temp_fn) as temp:
assert json.load(orig) == json.load(temp)
# Delete the temporary file
os.remove(temp_fn)
def test_write_source(self):
"""Test the write_source() method."""
req = Request()
for name in sample_data.keys():
orig_fn = self._filepath(name)
temp_fn = self._filepath(name + '-write-source')
# Read the message
resp = req.get(fromfile=orig_fn)
# Write to a temporary JSON file
resp.write_source(temp_fn)
# Read the two files and compare JSON (ignores ordering)
with open(orig_fn) as orig, open(temp_fn) as temp:
assert json.load(orig) == json.load(temp)
# Delete the temporary file
os.remove(temp_fn)
class TestStructSpecFlatDataSet(unittest.TestCase):
def setUp(self):
self.estat = Request('ESTAT')
filepath = os.path.join(
pkg_path, 'data/exr/ecb_exr_ng/structured/ecb_exr_ng_flat.xml')
dsd_resp = self.estat.datastructure(
fromfile=os.path.join(pkg_path, 'data/exr/ecb_exr_ng/ecb_exr_ng_full.xml'))
dsd = dsd_resp.datastructure.DataStructure
self.resp = self.estat.get(fromfile=filepath, dsd=dsd)
def test_msg_type(self):
self.assertIsInstance(self.resp.msg, model.DataMessage)
def test_header_attributes(self):
self.assertEqual(self.resp.header.structured_by, 'STR1')
self.assertEqual(self.resp.header.dim_at_obs, 'AllDimensions')
def test_dataset_cls(self):
self.assertIsInstance(self.resp.data, model.DataSet)
self.assertEqual(self.resp.msg.data.dim_at_obs, 'AllDimensions')
def test_generic_obs(self):
data = self.resp.data
# empty series list
self.assertEqual(len(list(data.series)), 0)
obs_list = list(data.obs())
self.assertEqual(len(obs_list), 12)
o0 = obs_list[0]
self.assertEqual(len(o0), 3)
self.assertIsInstance(o0.key, tuple) # obs_key
self.assertEqual(o0.key.FREQ, 'M')
self.assertEqual(o0.key.CURRENCY, 'CHF')
self.assertEqual(o0.value, '1.3413')
self.assertIsInstance(o0.attrib, tuple)
self.assertEqual(o0.attrib.OBS_STATUS, 'A')
self.assertEqual(o0.attrib.DECIMALS, '4')
def test_write2pandas(self):
data_series = self.resp.write(attributes='',
asframe=False, reverse_obs=False)
self.assertIsInstance(data_series, pandas.Series)
class TestStructSpecFlatDataSet(unittest.TestCase):
def setUp(self):
self.estat = Request('ESTAT')
filepath = os.path.join(
pkg_path, 'data/exr/ecb_exr_ng/structured/ecb_exr_ng_flat.xml')
dsd_resp = self.estat.datastructure(
fromfile=os.path.join(pkg_path, 'data/exr/ecb_exr_ng/ecb_exr_ng_full.xml'))
dsd = dsd_resp.datastructure.DataStructure
self.resp = self.estat.get(fromfile=filepath, dsd=dsd)
def test_msg_type(self):
self.assertIsInstance(self.resp.msg, model.DataMessage)
def test_header_attributes(self):
self.assertEqual(self.resp.header.structured_by, 'STR1')
self.assertEqual(self.resp.header.dim_at_obs, 'AllDimensions')
def test_dataset_cls(self):
self.assertIsInstance(self.resp.data, model.DataSet)
self.assertEqual(self.resp.msg.data.dim_at_obs, 'AllDimensions')
def test_generic_obs(self):
data = self.resp.data
# empty series list
self.assertEqual(len(list(data.series)), 0)
obs_list = list(data.obs())
self.assertEqual(len(obs_list), 12)
o0 = obs_list[0]
self.assertEqual(len(o0), 3)
self.assertIsInstance(o0.key, tuple) # obs_key
self.assertEqual(o0.key.FREQ, 'M')
self.assertEqual(o0.key.CURRENCY, 'CHF')
self.assertEqual(o0.value, '1.3413')
self.assertIsInstance(o0.attrib, tuple)
self.assertEqual(o0.attrib.OBS_STATUS, 'A')
self.assertEqual(o0.attrib.DECIMALS, '4')
def test_write2pandas(self):
data_series = self.resp.write(attributes='',
asframe=False, reverse_obs=False)
self.assertIsInstance(data_series, pandas.Series)
class TestGenericSeriesData_SiblingGroup_TS(unittest.TestCase):
def setUp(self):
self.estat = Request()
filepath = os.path.join(
pkg_path, 'data/exr/ecb_exr_sg/generic/ecb_exr_sg_ts.xml')
self.resp = self.estat.get(fromfile=filepath)
def test_groups(self):
data = self.resp.data
self.assertEqual(len(list(data.groups)), 4)
self.assertEqual(len(list(data.series)), 4)
g2 = list(data.groups)[2]
self.assertEqual(g2.key.CURRENCY, 'JPY')
self.assertEqual(
g2.attrib.TITLE, 'ECB reference exchange rate, Japanese yen/Euro')
# Check group attributes of a series
s = list(data.series)[0]
g_attrib = s.group_attrib
self.assertEqual(len(g_attrib), 1)
self.assertIsInstance(g_attrib, tuple)
self.assertEqual(len(g_attrib), 1)
def test_doc_index1():
"""First code example in index.rst."""
estat = Request("ESTAT")
flow_response = estat.dataflow("une_rt_a")
with pytest.raises(TypeError):
# This presumes the DataStructureDefinition instance can conduct a
# network request for its own content
structure_response = flow_response.dataflow.une_rt_a.structure(
request=True, target_only=False
)
# Same effect
structure_response = estat.get(
"datastructure", flow_response.dataflow.une_rt_a.structure.id
)
# Even better: Request.get(…) should examine the class and ID of the object
# structure = estat.get(flow_response.dataflow.une_rt_a.structure)
# Show some codelists
s = pandasdmx.to_pandas(structure_response)
expected = pd.Series(
{
"AT": "Austria",
"BE": "Belgium",
"BG": "Bulgaria",
"CH": "Switzerland",
"CY": "Cyprus",
},
name="GEO",
).rename_axis("CL_GEO")
# Codelists are converted to a DictLike
assert isinstance(s.codelist, DictLike)
# Same effect
assert_pd_equal(s.codelist["CL_GEO"].sort_index().head(), expected)
class Test_ESTAT_dsd_apro_mk_cola(unittest.TestCase):
def setUp(self):
self.estat = Request('ESTAT')
filepath = os.path.join(test_path, 'data/estat/apro_dsd.xml')
self.resp = self.estat.get(fromfile=filepath)
def test_codelists_keys(self):
self.assertEqual(len(self.resp.codelist), 6)
self.assertIsInstance(self.resp.codelist.CL_GEO, model.Codelist)
def test_codelist_name(self):
self.assertEqual(self.resp.msg.codelist.CL_GEO.UK.name.en,
'United Kingdom')
assert self.resp.codelist.CL_FREQ.name.en == 'FREQ'
def test_code_cls(self):
self.assertIsInstance(self.resp.codelist.CL_FREQ.D, model.Code)
def test_writer(self):
df = self.resp.write(rows='codelist')
self.assertEqual(df.shape, (79, 2))
def tearDown(self):
pass
class test_exr_constraints(unittest.TestCase):
def setUp(self):
self.ecb = Request('ecb')
filepath = os.path.join(test_path, 'data/exr_flow.xml')
self.resp = self.ecb.get(fromfile=filepath)
def test_constrained_codes(self):
m = self.resp.msg
self.assertEqual(m._dim_ids[0], 'FREQ')
self.assertEqual(len(m._dim_ids), 5)
self.assertEqual(len(m._dim_ids), 5)
self.assertEqual(len(m._dim_codes), 5)
self.assertEqual(len(m._attr_ids), 9)
self.assertEqual(len(m._attr_codes), 9)
self.assertEqual(m._attr_ids[-1], 'UNIT_MULT')
self.assertIn('5', m._attr_codes.UNIT_MULT)
self.assertIn('W', m._dim_codes.FREQ)
self.assertIn('W', m._dim_codes.FREQ)
self.assertEqual(len(m._constrained_codes), 14)
self.assertNotIn('W', m._constrained_codes.FREQ)
key = {'FREQ': ['W']}
self.assertTrue(m.in_codes(key))
self.assertFalse(m.in_constraints(key, raise_error=False))
self.assertRaises(ValueError, m.in_constraints, key)
self.assertTrue(m.in_constraints({'CURRENCY': ['CHF']}))
# test with invalid key
self.assertRaises(TypeError, m._in_constraints, {'FREQ': 'A'})
# structure writer with constraints
out = self.resp.write()
cl = out.codelist
self.assertEqual(cl.shape, (3555, 2))
# unconstrained codelists
out = self.resp.write(constraint=False)
cl = out.codelist
self.assertEqual(cl.shape, (4177, 2))
class test_dsd_common(unittest.TestCase):
def setUp(self):
self.estat = Request('ESTAT')
filepath = os.path.join(test_path, 'data/common/common.xml')
self.resp = self.estat.get(fromfile=filepath)
def test_codelists_keys(self):
self.assertEqual(len(self.resp.msg.codelist), 5)
self.assertIsInstance(self.resp.msg.codelist.CL_FREQ, model.Codelist)
def test_codelist_name(self):
self.assertEqual(self.resp.msg.codelist.CL_FREQ.D.name.en, 'Daily')
def test_code_cls(self):
self.assertIsInstance(self.resp.msg.codelist.CL_FREQ.D, model.Code)
def test_annotations(self):
code = self.resp.codelist.CL_FREQ.A
anno_list = list(code.annotations)
self.assertEqual(len(anno_list), 1)
a = anno_list[0]
self.assertIsInstance(a, model.Annotation)
self.assertTrue(a.text.en.startswith('It is'))
self.assertEqual(a.annotationtype, 'NOTE')
class InseeTestCase(unittest.TestCase):
# nosetests -s -v pandasdmx.tests.test_insee:InseeTestCase
def setUp(self):
unittest.TestCase.setUp(self)
self.sdmx = Request('INSEE')
def test_load_dataset(self):
dataset_code = 'IPI-2010-A21'
'''load all dataflows'''
dataflows_response = self.sdmx.get(
resource_type='dataflow', agency='FR1', fromfile=DATAFLOW_FP)
dataflows = dataflows_response.msg.dataflow
self.assertEqual(len(dataflows.keys()), 663)
self.assertTrue(dataset_code in dataflows)
'''load datastructure for current dataset_code'''
fp_datastructure = DATASETS[dataset_code]['datastructure-fp']
datastructure_response = self.sdmx.get(
resource_type='datastructure', agency='FR1', fromfile=fp_datastructure)
self.assertTrue(
dataset_code in datastructure_response.msg.datastructure)
dsd = datastructure_response.msg.datastructure[dataset_code]
'''Verify dimensions list'''
dimensions = OrderedDict([dim.id, dim] for dim in dsd.dimensions.aslist(
) if dim.id not in ['TIME', 'TIME_PERIOD'])
dim_keys = list(dimensions.keys())
self.assertEqual(dim_keys, ['FREQ', 'PRODUIT', 'NATURE'])
'''load datas for the current dataset'''
fp_data = DATASETS[dataset_code]['data-fp']
data = self.sdmx.get(
resource_type='data', agency='FR1', fromfile=fp_data)
'''Verify series count and values'''
series = list(data.msg.data.series)
series_count = len(series)
self.assertEqual(series_count, DATASETS[dataset_code]['series_count'])
first_series = series[0]
observations = list(first_series.obs())
first_obs = observations[0]
last_obs = observations[-1]
self.assertEqual(first_obs.dim, '2015-10')
self.assertEqual(first_obs.value, '105.61')
self.assertEqual(last_obs.dim, '1990-01')
self.assertEqual(last_obs.value, '139.22')
def test_fixe_key_names(self):
"""Verify key or attribute contains '-' in name
"""
dataset_code = 'CNA-2010-CONSO-SI-A17'
fp_datastructure = DATASETS[dataset_code]['datastructure-fp']
datastructure_response = self.sdmx.get(
resource_type='datastructure', agency='FR1', fromfile=fp_datastructure)
self.assertTrue(
dataset_code in datastructure_response.msg.datastructure)
dsd = datastructure_response.msg.datastructure[dataset_code]
dimensions = OrderedDict([dim.id, dim] for dim in dsd.dimensions.aslist(
) if dim.id not in ['TIME', 'TIME_PERIOD'])
dim_keys = list(dimensions.keys())
self.assertEqual(
dim_keys, ['SECT-INST', 'OPERATION', 'PRODUIT', 'PRIX'])
fp_data = DATASETS[dataset_code]['data-fp']
data = self.sdmx.get(
resource_type='data', agency='FR1', fromfile=fp_data)
series = list(data.msg.data.series)
series = series[0]
self.assertEqual(list(series.key._asdict().keys()),
['SECT-INST', 'OPERATION', 'PRODUIT', 'PRIX'])
self.assertEqual(list(series.attrib._asdict().keys()),
['FREQ', 'IDBANK', 'TITLE', 'LAST_UPDATE', 'UNIT_MEASURE', 'UNIT_MULT', 'REF_AREA', 'DECIMALS', 'BASE_PER', 'TIME_PER_COLLECT'])
def test_freq_in_series_attribute(self):
# Test that we don't have regression on Issues #39 and #41
# INSEE time series provide the FREQ value as attribute on the series instead of a dimension. This caused
# a runtime error when writing as pandas dataframe.
data_response = self.sdmx.data(
fromfile=SERIES['UNEMPLOYMENT_CAT_A_B_C']['data-fp'])
data_response.write()
dflows = flows.write().dataflow
# Listing tables from the high tech database and the description of a single table
ht_tabs = dflows[dflows.index.str.startswith('htec') == True]
kia_emp = dflows.loc['htec_kia_emp2'][0]
# Dataflow definition
df_def = flows.dataflow.htec_kia_emp2
# Database's datastructure id
dsd_id = df_def.structure.id
# Creating a support dict
refs = dict(references = 'all')
# Calling the table
dsd_response = estat.get(url = 'http://ec.europa.eu/eurostat/SDMX/diss-web/rest/datastructure/ESTAT/' + dsd_id)
# Getting informatou about the datastructure
dsd_response.url
dsd_response.http_headers
# Getting the datastructure of the table
dsd = dsd_response.datastructure[dsd_id]
# Dimensions and attributes
dsd.measures.aslist()
dsd.dimensions.aslist()
dsd.attributes.aslist()
# Getting dimension values
class InseeTestCase(unittest.TestCase):
# nosetests -s -v pandasdmx.tests.test_insee:InseeTestCase
def setUp(self):
unittest.TestCase.setUp(self)
self.sdmx = Request('INSEE')
def test_load_dataset(self):
dataset_code = 'IPI-2010-A21'
'''load all dataflows'''
dataflows_response = self.sdmx.get(
resource_type='dataflow', agency='FR1', fromfile=DATAFLOW_FP)
dataflows = dataflows_response.msg.dataflow
self.assertEqual(len(dataflows.keys()), 663)
self.assertTrue(dataset_code in dataflows)
'''load datastructure for current dataset_code'''
fp_datastructure = DATASETS[dataset_code]['datastructure-fp']
datastructure_response = self.sdmx.get(
resource_type='datastructure', agency='FR1', fromfile=fp_datastructure)
self.assertTrue(
dataset_code in datastructure_response.msg.datastructure)
dsd = datastructure_response.msg.datastructure[dataset_code]
'''Verify dimensions list'''
dimensions = OrderedDict([dim.id, dim] for dim in dsd.dimensions.aslist(
) if dim.id not in ['TIME', 'TIME_PERIOD'])
dim_keys = list(dimensions.keys())
self.assertEqual(dim_keys, ['FREQ', 'PRODUIT', 'NATURE'])
'''load datas for the current dataset'''
fp_data = DATASETS[dataset_code]['data-fp']
data = self.sdmx.get(
resource_type='data', agency='FR1', fromfile=fp_data)
'''Verify series count and values'''
series = list(data.msg.data.series)
series_count = len(series)
self.assertEqual(series_count, DATASETS[dataset_code]['series_count'])
first_series = series[0]
observations = list(first_series.obs())
first_obs = observations[0]
last_obs = observations[-1]
self.assertEqual(first_obs.dim, '2015-10')
self.assertEqual(first_obs.value, '105.61')
self.assertEqual(last_obs.dim, '1990-01')
self.assertEqual(last_obs.value, '139.22')
def test_fixe_key_names(self):
"""Verify key or attribute contains '-' in name
"""
dataset_code = 'CNA-2010-CONSO-SI-A17'
fp_datastructure = DATASETS[dataset_code]['datastructure-fp']
datastructure_response = self.sdmx.get(
resource_type='datastructure', agency='FR1', fromfile=fp_datastructure)
self.assertTrue(
dataset_code in datastructure_response.msg.datastructure)
dsd = datastructure_response.msg.datastructure[dataset_code]
dimensions = OrderedDict([dim.id, dim] for dim in dsd.dimensions.aslist(
) if dim.id not in ['TIME', 'TIME_PERIOD'])
dim_keys = list(dimensions.keys())
self.assertEqual(
dim_keys, ['SECT-INST', 'OPERATION', 'PRODUIT', 'PRIX'])
fp_data = DATASETS[dataset_code]['data-fp']
data = self.sdmx.get(
resource_type='data', agency='FR1', fromfile=fp_data)
series = list(data.msg.data.series)
series = series[0]
self.assertEqual(list(series.key._asdict().keys()),
['SECT-INST', 'OPERATION', 'PRODUIT', 'PRIX'])
self.assertEqual(list(series.attrib._asdict().keys()),
['FREQ', 'IDBANK', 'TITLE', 'LAST_UPDATE', 'UNIT_MEASURE', 'UNIT_MULT', 'REF_AREA', 'DECIMALS', 'BASE_PER', 'TIME_PER_COLLECT'])