class EurostatData(SeriesIterator): def __init__(self, dataset): super().__init__(dataset) self.dataset_url = make_url(self.dataset_code) self.xml_dsd = XMLStructure(provider_name=self.provider_name) self.xml_dsd.concepts = self.fetcher._concepts self.xml_dsd.codelists = self.fetcher._codelists self.store_path = self.get_store_path() self._load() def _load(self): download = Downloader(url=self.dataset_url, filename="data-%s.zip" % self.dataset_code, store_filepath=self.store_path, use_existing_file=self.fetcher.use_existing_file) filepaths = (extract_zip_file(download.get_filepath())) dsd_fp = filepaths[self.dataset_code + ".dsd.xml"] data_fp = filepaths[self.dataset_code + ".sdmx.xml"] self.fetcher.for_delete.append(dsd_fp) self.fetcher.for_delete.append(data_fp) self.xml_dsd.process(dsd_fp) self._set_dataset() self.xml_data = XMLData(provider_name=self.provider_name, dataset_code=self.dataset_code, xml_dsd=self.xml_dsd, dsd_id=self.dataset_code, #TODO: frequencies_supported=FREQUENCIES_SUPPORTED ) self.rows = self.xml_data.process(data_fp) def _set_dataset(self): dataset = dataset_converter(self.xml_dsd, self.dataset_code) self.dataset.dimension_keys = dataset["dimension_keys"] self.dataset.attribute_keys = dataset["attribute_keys"] self.dataset.concepts = dataset["concepts"] self.dataset.codelists = dataset["codelists"] def build_series(self, bson): self.dataset.add_frequency(bson["frequency"]) bson["last_update"] = self.dataset.last_update return bson
class EurostatData(SeriesIterator): def __init__(self, dataset): super().__init__(dataset) self.dataset_url = make_url(self.dataset_code) self.xml_dsd = XMLStructure(provider_name=self.provider_name) self.store_path = self.get_store_path() self._load() def _load(self): download = Downloader(url=self.dataset_url, filename="data-%s.zip" % self.dataset_code, store_filepath=self.store_path, use_existing_file=self.fetcher.use_existing_file) filepaths = (extract_zip_file(download.get_filepath())) dsd_fp = filepaths[self.dataset_code + ".dsd.xml"] data_fp = filepaths[self.dataset_code + ".sdmx.xml"] self.fetcher.for_delete.append(dsd_fp) self.fetcher.for_delete.append(data_fp) self.xml_dsd.process(dsd_fp) self._set_dataset() self.xml_data = XMLData( provider_name=self.provider_name, dataset_code=self.dataset_code, xml_dsd=self.xml_dsd, dsd_id=self.dataset_code, #TODO: frequencies_supported=FREQUENCIES_SUPPORTED ) self.rows = self.xml_data.process(data_fp) def _set_dataset(self): dataset = dataset_converter(self.xml_dsd, self.dataset_code) self.dataset.dimension_keys = dataset["dimension_keys"] self.dataset.attribute_keys = dataset["attribute_keys"] self.dataset.concepts = dataset["concepts"] self.dataset.codelists = dataset["codelists"] def build_series(self, bson): self.dataset.add_frequency(bson["frequency"]) bson["last_update"] = self.dataset.last_update return bson