def _load(self):
download = Downloader(url=self.dataset_url,
filename="data-%s.zip" % self.dataset_code,
store_filepath=self.store_path,
use_existing_file=self.fetcher.use_existing_file)
filepaths = (extract_zip_file(download.get_filepath()))
dsd_fp = filepaths[self.dataset_code + ".dsd.xml"]
data_fp = filepaths[self.dataset_code + ".sdmx.xml"]
self.fetcher.for_delete.append(dsd_fp)
self.fetcher.for_delete.append(data_fp)
self.xml_dsd.process(dsd_fp)
self._set_dataset()
self.xml_data = XMLData(
provider_name=self.provider_name,
dataset_code=self.dataset_code,
xml_dsd=self.xml_dsd,
dsd_id=self.dataset_code,
#TODO: frequencies_supported=FREQUENCIES_SUPPORTED
)
self.rows = self.xml_data.process(data_fp)
def _load(self):
download = Downloader(
url=self.dataset_url,
filename="data-%s.zip" % self.dataset_code,
store_filepath=self.store_path,
use_existing_file=self.fetcher.use_existing_file,
)
filepaths = extract_zip_file(download.get_filepath())
dsd_fp = filepaths[self.dataset_code + ".dsd.xml"]
data_fp = filepaths[self.dataset_code + ".sdmx.xml"]
self.fetcher.for_delete.append(dsd_fp)
self.fetcher.for_delete.append(data_fp)
self.xml_dsd.process(dsd_fp)
self._set_dataset()
self.xml_data = XMLData(
provider_name=self.provider_name,
dataset_code=self.dataset_code,
xml_dsd=self.xml_dsd,
dsd_id=self.dataset_code,
# TODO: frequencies_supported=FREQUENCIES_SUPPORTED
)
self.rows = self.xml_data.process(data_fp)
class EurostatData(SeriesIterator):
def __init__(self, dataset):
super().__init__(dataset)
self.dataset_url = make_url(self.dataset_code)
self.xml_dsd = XMLStructure(provider_name=self.provider_name)
self.xml_dsd.concepts = self.fetcher._concepts
self.xml_dsd.codelists = self.fetcher._codelists
self.store_path = self.get_store_path()
self._load()
def _load(self):
download = Downloader(url=self.dataset_url,
filename="data-%s.zip" % self.dataset_code,
store_filepath=self.store_path,
use_existing_file=self.fetcher.use_existing_file)
filepaths = (extract_zip_file(download.get_filepath()))
dsd_fp = filepaths[self.dataset_code + ".dsd.xml"]
data_fp = filepaths[self.dataset_code + ".sdmx.xml"]
self.fetcher.for_delete.append(dsd_fp)
self.fetcher.for_delete.append(data_fp)
self.xml_dsd.process(dsd_fp)
self._set_dataset()
self.xml_data = XMLData(provider_name=self.provider_name,
dataset_code=self.dataset_code,
xml_dsd=self.xml_dsd,
dsd_id=self.dataset_code,
#TODO: frequencies_supported=FREQUENCIES_SUPPORTED
)
self.rows = self.xml_data.process(data_fp)
def _set_dataset(self):
dataset = dataset_converter(self.xml_dsd, self.dataset_code)
self.dataset.dimension_keys = dataset["dimension_keys"]
self.dataset.attribute_keys = dataset["attribute_keys"]
self.dataset.concepts = dataset["concepts"]
self.dataset.codelists = dataset["codelists"]
def build_series(self, bson):
self.dataset.add_frequency(bson["frequency"])
bson["last_update"] = self.dataset.last_update
return bson
class EurostatData(SeriesIterator):
def __init__(self, dataset):
super().__init__(dataset)
self.dataset_url = make_url(self.dataset_code)
self.xml_dsd = XMLStructure(provider_name=self.provider_name)
self.store_path = self.get_store_path()
self._load()
def _load(self):
download = Downloader(url=self.dataset_url,
filename="data-%s.zip" % self.dataset_code,
store_filepath=self.store_path,
use_existing_file=self.fetcher.use_existing_file)
filepaths = (extract_zip_file(download.get_filepath()))
dsd_fp = filepaths[self.dataset_code + ".dsd.xml"]
data_fp = filepaths[self.dataset_code + ".sdmx.xml"]
self.fetcher.for_delete.append(dsd_fp)
self.fetcher.for_delete.append(data_fp)
self.xml_dsd.process(dsd_fp)
self._set_dataset()
self.xml_data = XMLData(
provider_name=self.provider_name,
dataset_code=self.dataset_code,
xml_dsd=self.xml_dsd,
dsd_id=self.dataset_code,
#TODO: frequencies_supported=FREQUENCIES_SUPPORTED
)
self.rows = self.xml_data.process(data_fp)
def _set_dataset(self):
dataset = dataset_converter(self.xml_dsd, self.dataset_code)
self.dataset.dimension_keys = dataset["dimension_keys"]
self.dataset.attribute_keys = dataset["attribute_keys"]
self.dataset.concepts = dataset["concepts"]
self.dataset.codelists = dataset["codelists"]
def build_series(self, bson):
self.dataset.add_frequency(bson["frequency"])
bson["last_update"] = self.dataset.last_update
return bson