def test_request():
# Constructor
r = pandasdmx.Request(log_level=logging.ERROR)
# Invalid source name raise an exception
with pytest.raises(ValueError):
pandasdmx.Request("noagency")
# Regular methods
r.clear_cache()
r.timeout = 300
assert r.timeout == 300
# dir() includes convenience methods for resource endpoints
expected = {
"cache",
"clear_cache",
"get",
"preview_data",
"series_keys",
"session",
"source",
"timeout",
"view_doc",
}
expected |= set(ep.name for ep in pandasdmx.Resource)
assert set(filter(lambda s: not s.startswith("_"), dir(r))) == expected
def fetch_dataset(db_collection, flow_name: str, parameters):
# 1. try CSV fetch, if that doesnt work then try PandasSDMX to get the dataframe
data_response = requests.get(url,
params=parameters,
headers={"Accept": "text/csv"})
assert data_response.status_code == 200
with get_tempfile() as fp:
fp.write(data_response.text.encode())
fp.seek(0)
kwargs = ({} if not flow_name in extra_csv_parms else dict(
**extra_csv_parms[flow_name]))
try:
df = pd.read_csv(fp, **kwargs)
save_dataframe(db_collection, {}, df, url, "ECB")
return
except pd.errors.EmptyDataError: # no data is ignored as far as --fail-fast is concerned
print(
f"No CSV data to save.. now trying {flow_name} using pandasdmx"
)
# FALLTHRU...
# 2. try pandassdmx if CSV fetch fails
ecb = sdmx.Request("ECB", backend="memory")
data_msg = ecb.data(flow_name, params=parameters)
df = sdmx.to_pandas(data_msg)
assert isinstance(df, pd.DataFrame)
save_dataframe(db_collection, {}, df, url, "ECB")
def downloadCSV():
try:
script_dir = os.path.abspath(os.path.dirname(sys.argv[0]) or '.')
csv_path = os.path.join(script_dir, 'DATA_CSV')
oecd = pandasdmx.Request('OECD')
while 1:
lock.acquire()
if len(datasetcode) > 0:
code = datasetcode.pop(0)
title = filename.pop(0)
lock.release()
print('in datasetcode=', code, "length of datasetcode ",
len(datasetcode))
try:
data_response = oecd.data(resource_id=code, key='all')
df = data_response.write(data_response.data.series,
parse_time=False)
s = str(title)
while s[-1] == ".":
s = s[:-1]
df.to_csv(csv_path + '\\' + s + '.csv', sep=',')
print('in download, completed to_csv', "datacode=", code)
except:
pass
else:
lock.release()
time.sleep(0.01)
except:
pass
def test_request_get_args():
req = sdmx.Request('ESTAT')
# Request._make_key accepts '+'-separated values
args = dict(resource_id='une_rt_a',
key={'GEO': 'EL+ES+IE'},
params={'startPeriod': '2007'},
dry_run=True,
use_cache=True)
# Store the URL
url = req.data(**args).url
# Using an iterable of key values gives the same URL
args['key'] = {'GEO': ['EL', 'ES', 'IE']}
assert req.data(**args).url == url
# Using a direct string for a key gives the same URL
args['key'] = '....EL+ES+IE' # No specified values for first 4 dimensions
assert req.data(**args).url == url
# Giving 'provider' is redundant for a data request, causes a warning
with pytest.warns(UserWarning, match="'provider' argument is redundant"):
req.data('une_rt_a',
key={'GEO': 'EL+ES+IE'},
params={'startPeriod': '2007'},
provider='ESTAT')
# Using an unknown endpoint is an exception
with pytest.raises(ValueError):
req.get('badendpoint', 'id')
def test_write_constraint():
"""'constraint' argument to writer.write_dataset."""
with specimen('ng-ts.xml') as f:
msg = sdmx.read_sdmx(f)
# Fetch the message's DSD
assert msg.structure.is_external_reference
# NB the speciment included in tests/data has 'ECB_EXR_NG' as the
# data structure ID; but a query against the web service gives
# 'ECB_EXR1' for the same data structure.
id = 'ECB_EXR1'
dsd = sdmx.Request(msg.structure.maintainer.id) \
.get('datastructure', id) \
.structure[id]
# Create a ContentConstraint
cc = dsd.make_constraint({'CURRENCY': 'JPY+USD'})
# Write the message without constraint
s1 = sdmx.to_pandas(msg)
assert len(s1) == 12
assert set(s1.index.to_frame()['CURRENCY']) == {'CHF', 'GBP', 'JPY', 'USD'}
# Writing using constraint produces a fewer items; only those matching the
# constraint
s2 = sdmx.to_pandas(msg, constraint=cc)
assert len(s2) == 6
assert set(s2.index.to_frame()['CURRENCY']) == {'JPY', 'USD'}
def test_write_constraint():
"""'constraint' argument to writer.write_dataset."""
with specimen("ng-ts.xml") as f:
msg = pandasdmx.read_sdmx(f)
# Fetch the message's DSD
assert msg.structure.is_external_reference
# NB the speciment included in tests/data has 'ECB_EXR_NG' as the
# data structure ID; but a query against the web service gives
# 'ECB_EXR1' for the same data structure.
id = "ECB_EXR1"
dsd = (
pandasdmx.Request(msg.structure.maintainer.id)
.get("datastructure", id)
.structure[id]
)
# Create a ContentConstraint
cc = dsd.make_constraint({"CURRENCY": "JPY+USD"})
# Write the message without constraint
s1 = pandasdmx.to_pandas(msg)
assert len(s1) == 12
assert set(s1.index.to_frame()["CURRENCY"]) == {"CHF", "GBP", "JPY", "USD"}
# Writing using constraint produces a fewer items; only those matching the
# constraint
s2 = pandasdmx.to_pandas(msg, constraint=cc)
assert len(s2) == 6
assert set(s2.index.to_frame()["CURRENCY"]) == {"JPY", "USD"}
def test_request_get_args():
req = pandasdmx.Request("ESTAT")
# Request._make_key accepts '+'-separated values
args = dict(
resource_id="une_rt_a",
key={"GEO": "EL+ES+IE"},
params={"startPeriod": "2007"},
dry_run=True,
use_cache=True,
)
# Store the URL
url = req.data(**args).url
# Using an iterable of key values gives the same URL
args["key"] = {"GEO": ["EL", "ES", "IE"]}
assert req.data(**args).url == url
# Using a direct string for a key gives the same URL
args["key"] = "....EL+ES+IE" # No specified values for first 4 dimensions
assert req.data(**args).url == url
# Giving 'provider' is redundant for a data request, causes a warning
with pytest.warns(UserWarning, match="'provider' argument is redundant"):
req.data(
"une_rt_a",
key={"GEO": "EL+ES+IE"},
params={"startPeriod": "2007"},
provider="ESTAT",
)
# Using an unknown endpoint is an exception
with pytest.raises(ValueError):
req.get("badendpoint", "id")
def _download_sdmx(table: str, dst_dir: PathLike) -> None:
path = dst_dir / _SDMX_FILENAME
service = conf["sdmx_service_name"]
name = conf["sdmx_datastructure_template"].format(table=table)
r = pandasdmx.Request(service)
r.datastructure(name, tofile=path)
def create_estat_request():
# EuroStat datasets
if 'CACHE_FILE_LOCATION' in app.config:
cache_name = app.config['CACHE_FILE_LOCATION']
else:
cache_name = "/tmp/sdmx_datasets_cache"
r = pandasdmx.Request("ESTAT", cache={"backend": "sqlite", "include_get_headers": True,
"cache_name": cache_name})
r.timeout = 180
return r
def test_request_get_exceptions():
"""Tests of Request.get() that don't require remote data."""
req = pandasdmx.Request("ESTAT")
# Exception is raised on unrecognized arguments
exc = "unrecognized arguments: {'foo': 'bar'}"
with pytest.raises(ValueError, match=exc):
req.get("datastructure", foo="bar")
with pytest.raises(ValueError, match=exc):
pandasdmx.read_url("https://example.com", foo="bar")
def test_request_get_exceptions():
"""Tests of Request.get() that don't require remote data."""
req = sdmx.Request('ESTAT')
# Exception is raised on unrecognized arguments
exc = "unrecognized arguments: {'foo': 'bar'}"
with pytest.raises(ValueError, match=exc):
req.get('datastructure', foo='bar')
with pytest.raises(ValueError, match=exc):
sdmx.read_url('https://example.com', foo='bar')
def test_request():
# Constructor
r = sdmx.Request(log_level=logging.ERROR)
# Invalid source name raise an exception
with pytest.raises(ValueError):
sdmx.Request('noagency')
# Regular methods
r.clear_cache()
r.timeout = 300
assert r.timeout == 300
# dir() includes convenience methods for resource endpoints
expected = {
'cache', 'clear_cache', 'get', 'preview_data', 'series_keys',
'session', 'source', 'timeout'
}
expected |= set(ep.name for ep in sdmx.Resource)
assert set(filter(lambda s: not s.startswith('_'), dir(r))) == expected
def _read_structure_message(path):
# Need to support pandasdmx==0.9 because 1.0 is not available for Python3.6
if pandasdmx.__version__.startswith("0.9"):
req = pandasdmx.Request()
structure = req.get(fromfile=str(path),
writer="pandasdmx.writer.structure2pd")
return structure.write()
if pandasdmx.__version__.startswith("1."):
return pandasdmx.read_sdmx(path)
raise Exception(f"pandasdmx version is {pandasdmx.__version__}")
def fx_rates(cur_from, cur_to, freq = "A", start_year = "2015"):
if cur_from == "EUR":
request_ecb = sdmx.Request("ECB")
fx_rates = (
request_ecb.data(resource_id="EXR", key={'CURRENCY': cur_to, "FREQ": freq, "EXR_SUFFIX": "A"}, params = {'startPeriod': start_year})
.to_pandas()
.reset_index()
.loc[:, ["CURRENCY", "CURRENCY_DENOM", "TIME_PERIOD", "value"]]
.set_axis(["to", "from", "period", "value"], axis=1)
)
if "EUR" not in set(fx_rates["to"]) and "EUR" in cur_to:
dummy_fx_eur = (
fx_rates[["period", "from"]]
.assign(to=lambda x: "EUR")
.assign(value=lambda x: 1)
)
fx_rates = fx_rates.append(dummy_fx_eur, ignore_index=True)
if cur_from == "USD":
request_bis = sdmx.Request("BIS")
fx_rates = (
request_bis.data(resource_id = "WEBSTATS_XRU_CURRENT_DATAFLOW", key={"FREQ": freq, "CURRENCY": cur_to, "COLLECTION": "A"}, params={"startPeriod": start_year})
.to_pandas()
.reset_index()
.loc[:, ["CURRENCY", "TIME_PERIOD", "value"]]
.assign(from_name=lambda x: "USD")
.set_axis(["to", "period", "value" ,"from"], axis=1)
)
fx_rates = (
fx_rates
.groupby(["period", "from", "to"])
.mean()
.reset_index()
)
return(fx_rates)
def get_uk_regional_gdp():
# current year
latest_year = datetime.now().year - 1
# Tell pdmx we want OECD data
oecd = pdmx.Request("OECD")
# Set out everything about the request in the format specified by the OECD API
data = oecd.data(
resource_id="REGION_ECONOM",
key="1+2.UKC.SNA_2008.GDP.REG+CURR_PR.ALL.2017+2018+2019+2020/all?",
).to_pandas()
# example that works:
"https://stats.oecd.org/restsdmx/sdmx.ashx/GetData/REGION_ECONOM/1+2.GBR+UKC+UKC11+UKC12.SNA_2008.GDP.REG+CURR_PR+USD_PPP+REAL_PR+REAL_PPP+PC+PC_CURR_PR+PC_USD_PPP+PC_REAL_PR+PC_REAL_PPP.ALL.2001+2002+2003+2004+2005+2006+2007+2008+2009+2010+2011+2012+2013+2014+2015+2016+2017+2018+2019+2020/all?"
df = pd.DataFrame(data).reset_index()
df.head()
def create_estat_request():
# from magic_box import app # APP is this now. Adapt to the current app object!
app = import_names("backend.restful_service", "app")
if not app:
app = import_names("magic_box", "app")
# EuroStat datasets
if 'CACHE_FILE_LOCATION' in app.config:
cache_name = app.config['CACHE_FILE_LOCATION']
print("USER: "******"/sdmx_datasets_cache"
r = pandasdmx.Request("ESTAT", cache={"backend": "sqlite", "include_get_headers": True, "cache_name": cache_name})
r.timeout = 180
return r
def _read(the_dir: PathLike) -> Mapping[str, pd.DataFrame]:
path = the_dir / _SDMX_FILENAME
req = pandasdmx.Request()
structure = req.get(
fromfile=str(path),
writer='pandasdmx.writer.structure2pd'
)
codelist = structure.write()['codelist']
result = {
'dimensions': _get_dimensions(codelist),
'attributes': _get_attributes(codelist),
}
return result
def create_estat_request():
# EuroStat datasets
if get_global_configuration_variable('CACHE_FILE_LOCATION'):
cache_name = get_global_configuration_variable('CACHE_FILE_LOCATION')
print("USER: "******"/sdmx_datasets_cache"
r = pandasdmx.Request("ESTAT",
cache={
"backend": "sqlite",
"include_get_headers": True,
"cache_name": cache_name
})
r.timeout = 180
return r
def test_request_preview_data():
req = sdmx.Request('ECB')
# List of keys can be retrieved
keys = req.preview_data('EXR')
assert isinstance(keys, list)
# Count of keys can be determined
assert len(keys) == 4291
# A filter can be provided, resulting in fewer keys
keys = req.preview_data('EXR', {'CURRENCY': 'CAD+CHF+CNY'})
assert len(keys) == 24
# Result can be converted to pandas object
keys_pd = sdmx.to_pandas(keys)
assert isinstance(keys_pd, pd.DataFrame)
assert len(keys_pd) == 24
def test_request_preview_data():
req = pandasdmx.Request("ECB")
# List of keys can be retrieved
keys = req.preview_data("EXR")
assert isinstance(keys, list)
# Count of keys can be determined
assert len(keys) > 1000
# A filter can be provided, resulting in fewer keys
keys = req.preview_data("EXR", {"CURRENCY": "CAD+CHF+CNY"})
assert len(keys) == 24
# Result can be converted to pandas object
keys_pd = pandasdmx.to_pandas(keys)
assert isinstance(keys_pd, pd.DataFrame)
assert len(keys_pd) == 24
def ecb():
ecb = sdmx.Request('ECB',
backend='sqlite',
fast_save=True,
expire_after=6000)
dump_flows(ecb)
dsd = dump_structure(ecb, 'YC')
for dim_name in ['FREQ', 'REF_AREA', 'DATA_TYPE_FM']:
results = get_available_of_dimension(dsd, dim_name)
print(results)
# external trade for australia since 2000
dsd = dump_structure(ecb, 'TRD')
print("*** External Trade")
dump_dimension(dsd, 'REF_AREA', all=True)
dump_dimension(dsd, 'TRD_PRODUCT')
dump_dimension(dsd, 'TRD_FLOW')
resp = ecb.data('TRD', key={'REF_AREA': 'I8'})
df = resp.to_pandas()
print(df)
def test_doc_example():
"""Code from example.rst."""
import pandasdmx
estat = pandasdmx.Request("ESTAT")
metadata = estat.datastructure("DSD_une_rt_a")
for cl in "CL_AGE", "CL_UNIT":
print(pandasdmx.to_pandas(metadata.codelist[cl]))
resp = estat.data("une_rt_a",
key={"GEO": "EL+ES+IE"},
params={"startPeriod": "2007"})
data = pandasdmx.to_pandas(resp,
datetime=dict(dim='TIME_PERIOD',
freq='FREQ')).xs("Y15-74",
axis=1,
level="AGE",
drop_level=False)
data.loc[:, ("Y15-74", "PC_ACT", "T")]
# Further checks per https://github.com/dr-leo/pandaSDMX/issues/157
# DimensionDescriptor for the structure message
dd1 = metadata.structure.DSD_une_rt_a.dimensions
# DimensionDescriptor retrieved whilst validating the data message
dd2 = resp.data[0].structured_by.dimensions
# DimensionDescriptors have same ID, components and order
assert dd1 == dd2
# One SeriesKey from the data message
sk = list(resp.data[0].series.keys())[0]
# Key values have same order as in the DSD
assert dd1.order_key(sk) == sk
def download_source():
"""
Downloads the following datastructure from the UN SDMX API
(1) United Nations Energy Balance Data
(2) United Nations Greenhouse gas Emission Data
Data is stored at corresponding CSV files
"""
#if os.path.exists(UNFCC_FILE):
# os.rename(UNFCC_FILE,'old_'+UNFCC_FILE)
#if os.path.exists(EBAL_FILE):
# os.rename(EBAL_FILE,'old_'+EBAL_FILE)
try:
unsd = sdmx.Request('UNSD')
sdmx.logger.setLevel(logging.INFO)
logger.info('Loading UNFCC Data')
resp_unfcc = unsd.data('DF_UNData_UNFCC')
logger.info('Loading UN Energy Balance Data')
resp_ebal = unsd.data('DF_UNData_EnergyBalance')
except Exception as e:
logger.error('Error!! Please look at SDMX logs to troubleshoot' +
str(e))
traceback.print_exc(file=sys.stdout)
try:
df_ebal = resp_ebal.to_pandas()
df_unfcc = resp_unfcc.to_pandas()
df_unfcc.reset_index().to_csv(UNFCC_FILE, index=False)
logger.info('UNFCC Greenhouse Data stored as {}'.format(UNFCC_FILE))
df_ebal.reset_index().to_csv(EBAL_FILE, index=False)
logger.info('UN Energy Balance Data stored as {}'.format(EBAL_FILE))
except Exception as e:
logger.error('Error!! While saving data from SDMX to CSV ' + str(e))
traceback.print_exc(file=sys.stdout)
def test_doc_example():
"""Code from example.rst."""
import pandasdmx as sdmx
estat = sdmx.Request('ESTAT')
metadata = estat.datastructure('DSD_une_rt_a')
for cl in 'CL_AGE', 'CL_UNIT':
print(sdmx.to_pandas(metadata.codelist[cl]))
resp = estat.data(
'une_rt_a',
key={'GEO': 'EL+ES+IE'},
params={'startPeriod': '2007'},
)
data = sdmx.to_pandas(resp) \
.xs('Y15-74', level='AGE', drop_level=False)
data.loc[('A', 'Y15-74', 'PC_ACT', 'T')]
# Further checks per https://github.com/dr-leo/pandaSDMX/issues/157
# DimensionDescriptor for the structure message
dd1 = metadata.structure.DSD_une_rt_a.dimensions
# DimensionDescriptor retrieved whilst validating the data message
dd2 = resp.data[0].structured_by.dimensions
# DimensionDescriptors have same ID, components and order
assert dd1 == dd2
# One SeriesKey from the data message
sk = list(resp.data[0].series.keys())[0]
# Key values have same order as in the DSD
assert dd1.order_key(sk) == sk
import pandasdmx as sdmx
# TODO: Move all of these to env/setting vars from production
sdmx_url = "https://sdmx.data.unicef.org/ws/public/sdmxapi/rest/data/ECARO,TRANSMONEE,1.0/.{}....?format=csv&startPeriod={}&endPeriod={}"
geo_json_file = (pathlib.Path(__file__).parent.parent.absolute() /
"assets/countries.geo.json")
with open(geo_json_file) as shapes_file:
geo_json_countries = json.load(shapes_file)
with open(
pathlib.Path(__file__).parent.parent.absolute() /
"assets/indicator_config.json") as config_file:
indicators_config = json.load(config_file)
unicef = sdmx.Request("UNICEF")
metadata = unicef.dataflow("TRANSMONEE", provider="ECARO", version="1.0")
dsd = metadata.structure["DSD_ECARO_TRANSMONEE"]
indicator_names = {
code.id: code.name.en
for code in dsd.dimensions.get("INDICATOR").local_representation.enumerated
}
# lbassil: get the age groups code list as it is not in the DSD
cl_age = unicef.codelist("CL_AGE", version="1.0")
age_groups = sdmx.to_pandas(cl_age)
dict_age_groups = age_groups["codelist"]["CL_AGE"].reset_index()
age_groups_names = {
age["CL_AGE"]: age["name"]
for index, age in dict_age_groups.iterrows() if age["CL_AGE"] != "_T"
# -*- coding: utf-8 -*-
"""
Created on Sun Apr 26 15:47:39 2020
@author: rohan
"""
import pandasdmx as sdmx
Agency_Code = 'ABS'
Dataset_Id = 'HF' #'ATSI_BIRTHS_SUMM'
ABS = sdmx.Request(Agency_Code)
data_response = ABS.data(resource_id=Dataset_Id,
params={
'startPeriod': '2018-11',
'endPeriod': '2018-11'
})
data = data_response.data
msg = data_response.msg
measure = msg._elem['structure']['dimensions']['series'][0]['values']
region = msg._elem['structure']['dimensions']['series'][1]['values']
lender = msg._elem['structure']['dimensions']['series'][2]['values']
adjustment = msg._elem['structure']['dimensions']['series'][3]['values']
item = msg._elem['structure']['dimensions']['series'][4]['values']
frequency = msg._elem['structure']['dimensions']['series'][5]['values']
def flatten(l):
return flatten(l[0]) + (
flatten(l[1:]) if len(l) > 1 else []) if type(l) is list else [l]
}, {
'id': 'TTL',
'name': 'Total'
}, {
'id': 'VAL',
'name': 'Value added'
}, {
'id': 'DOMIMP',
'name': 'Domestic output and imports'
}, {
'id': 'ICESHR',
'name': 'Imports content of exports, as % of exports'
}]
# Obtain some country data relevant for ecological footprint calculations
unstats = pandasdmx.Request("UNSD")
for name, description in [
("AG_LND_AGRI_ZS", "Agricultural land (% of land area)"),
("AG_LND_FRST_K2", "Forest area (sq. km)"),
("EN_ATM_CO2E_PC", "CO2 emissions (metric tons per capita)"),
("NY_GDP_MKTP_CD", "GDP (current US$)"),
("NY_GNP_PCAP_CD", "GNI per capita, Atlas method (current US$)"),
("NY_GNP_PCAP_PP_CD", "GNI per capita, PPP (current international $)"),
("NY_GNP_ATLS_CD", "GNI, Atlas method (current US$)"),
("NY_GNP_MKTP_PP_CD", "GNI, PPP (current international $)"),
("AG_SRF_TOTL_K2", "Surface area (sq. km)")
]:
data = unstats.get(resource_type='data',
resource_id="DF_UNDATA_WDI",
key={
"FREQ": "A",
"--dataset",
help="Download only the specified flowRef [None]",
type=str,
default=None,
)
a = args.parse_args()
config, password = read_config(a.config)
m = config.get("mongo")
mongo = pymongo.MongoClient(m.get("host"),
m.get("port"),
username=m.get("user"),
password=password)
db = mongo[m.get("db")]
ecb = sdmx.Request("ECB")
all_flows = ecb.dataflow()
df = sdmx.to_pandas(all_flows.dataflow).to_frame()
print(f"Found {len(df)} ECB datasets to process")
if a.dataset:
for s in a.dataset.split(","):
do_download.add(s)
recent_tags = set() # forcibly re-download all stated datasets
else:
month_ago = now() - timedelta(days=30)
recent_tags = set(
db.ecb_data_cache.distinct("tag",
{"last_updated": {
"$gte": month_ago
def OECD_dataset(dataset_ids, Logger):
Logger.warning('Trying to get data form OECD.stat')
oecd = pandasdmx.Request('OECD')
params = dict(startPeriod='1990-Q1',
endPeriod='2021-Q1',
dimensionAtObservation='TimeDimension',
detail='Full')
dfs = {}
# st.text(f'Getting Datasets names..')
# st.text(f'dataset ids: {dataset_ids}')
try:
for dataset in dataset_ids:
if 'GBARD' in dataset:
# st.text(f'Getting Dataset: {dataset}')
try:
Logger.warning(f'Loading {dataset} from OECD database')
data_response = oecd.data(resource_id=dataset,
params=params)
df = get_df(data_response, Logger)
Logger.debug(f'flattening {dataset} df')
df.reset_index(level=['COUNTRY', 'SEO', 'MEASURE'],
inplace=True)
sort_df: object = df.melt(
id_vars=['COUNTRY', 'SEO', 'MEASURE'],
var_name="Date",
value_name="Value")
dfs['gbard'] = sort_df
except req.exceptions.ConnectionError as err:
Logger.warning(f'This {dataset} got this error: {err}')
print(f'This dataset: {dataset} as failed to response')
continue
elif 'HIGH_AGLINK' in dataset:
# st.text(f'Getting Dataset: {dataset}')
try:
Logger.warning(f'Loading {dataset} from OECD database')
data_response = oecd.data(resource_id=dataset,
params=params)
df = get_df(data_response, Logger)
Logger.debug(f'flattening {dataset} df')
df.reset_index(level=['LOCATION', 'COMMODITY', 'VARIABLE'],
inplace=True)
sort_df: object = df.melt(
id_vars=['LOCATION', 'COMMODITY', 'VARIABLE'],
var_name="Date",
value_name="Value")
dfs['agricultural'] = sort_df
except req.exceptions.ConnectionError as err:
Logger.warning(f'This {dataset} got this error: {err}')
print(f'This dataset: {dataset} as failed to response')
continue
elif 'SNA_TABLE4' in dataset:
# st.text(f'Getting Dataset: {dataset}')
try:
Logger.warning(f'Loading {dataset} from OECD database')
data_response = oecd.data(resource_id=dataset,
params=params)
df = get_df(data_response, Logger)
Logger.debug(f'flattening {dataset} df')
df.reset_index(level=['LOCATION', 'TRANSACT', 'MEASURE'],
inplace=True)
sort_df: object = df.melt(
id_vars=['LOCATION', 'TRANSACT', 'MEASURE'],
var_name="Date",
value_name="Value")
dfs['currncy'] = sort_df
except req.exceptions.ConnectionError as err:
Logger.warning(f'This {dataset} got this error: {err}')
print(f'This dataset: {dataset} as failed to response')
continue
if len(dfs) == len(dataset_ids):
# st.text(f'Success of Getting Datasets')
Logger.debug('Success')
print('Success')
else:
# st.text(f'Only {len(dfs)} dataset were found: {list(dfs.keys())}')
Logger.info(f'Only {len(dfs)} dataset were found')
print(f'Only {len(dfs)} dataset were found')
return dfs
except TypeError:
pass
def _download_sdmx(table: str, dst_dir: PathLike) -> None:
path = dst_dir / _SDMX_FILENAME
service = conf['sdmx_service_name']
name = conf['sdmx_datastructure_template'].format(table=table)
r = pandasdmx.Request(service)
r.datastructure(name).write_source(str(path))
