def write(self): try: self.csv_to_xlsx() except IOError as e: catalog = self.csv_dump_file.node or 'global' msg = f"Error escribiendo dump XLSX de dump {catalog} {self.csv_dump_file.file_name}: {e.__class__}: {e}" GenerateDumpTask.info(self.task, msg)
def generate(self): if not self.fields: GenerateDumpTask.info(self.task, f"No hay series cargadas para el catálogo {self.catalog}") return FullCsvGenerator(self.task, self.fields, self.catalog).generate() ValuesCsvGenerator(self.task, self.fields, self.catalog).generate() SourcesCsvGenerator(self.task, self.fields, self.catalog).generate() MetadataCsvGenerator(self.task, self.fields, self.catalog).generate()
def enqueue_dump_task(task: GenerateDumpTask): task.save() # Evito problemas de DoesNotExist cuando se llama async task_choices = { GenerateDumpTask.TYPE_CSV: write_csv, GenerateDumpTask.TYPE_XLSX: write_xlsx, GenerateDumpTask.TYPE_SQL: write_sql, GenerateDumpTask.TYPE_DTA: write_dta, } task_choices[task.file_type](task.id)
def write_distribution(self, distribution: Distribution, writer: csv.writer): # noinspection PyBroadException try: df = read_distribution_csv(distribution) fields = distribution.field_set.all() fields = {field.title: field.identifier for field in fields} periodicity = meta_keys.get(distribution, meta_keys.PERIODICITY) df.apply(self.write_serie, args=(periodicity, fields, writer)) except Exception as e: msg = f'[{self.tag} Error en la distribución {distribution.identifier}: {e.__class__}: {e}' GenerateDumpTask.info(self.task, msg) logger.error(msg)
def test_leading_nulls_distribution(self): path = os.path.join(samples_dir, 'leading_nulls_distribution.json') index_catalog('leading_null', path, self.index) self.task = GenerateDumpTask() self.task.save() gen = DumpGenerator(self.task, 'leading_null') gen.generate() file = self.task.dumpfile_set.get(file_name=DumpFile.FILENAME_FULL, file_type=DumpFile.TYPE_CSV, node__catalog_id='leading_null').file reader = read_file_as_csv(file) next(reader) # Header!!!! self.assertEqual(len(list(reader)), 2)
def write_distribution(self, distribution: Distribution, writer: csv.writer): # noinspection PyBroadException try: fields = distribution.field_set.all() fields = {field.title: field.identifier for field in fields} periodicity = meta_keys.get(distribution, meta_keys.PERIODICITY) index_col = DistributionRepository( distribution).get_time_index_series().title df = DistributionCsvReader(distribution, index_col).read() df.apply(self.write_serie, args=(periodicity, fields, writer)) except Exception as e: msg = f'[{self.tag} Error en la distribución {distribution.identifier}: {e.__class__}: {e}' GenerateDumpTask.info(self.task, msg) logger.warning(msg)
def setUpClass(cls): super(CSVTest, cls).setUpClass() cls.catalog_id = 'csv_dump_test_catalog' path = os.path.join(samples_dir, 'distribution_daily_periodicity.json') index_catalog(cls.catalog_id, path, cls.index) cls.task = GenerateDumpTask() cls.task.save() gen = DumpGenerator(cls.task) gen.generate()
def test_invalid_catalog(self): task = GenerateDumpTask() task.save() gen = DumpGenerator(task, "no_catalog") gen.generate() task.refresh_from_db() self.assertFalse(DumpFile.objects.filter(file_type=DumpFile.TYPE_CSV, task=task))
def test_dump_distribution_no_periodicity(self): task = GenerateDumpTask() task.save() Distribution.objects.first().enhanced_meta.get(key=meta_keys.PERIODICITY).delete() gen = DumpGenerator(task) gen.generate() task.refresh_from_db() self.assertTrue(DumpFile.objects.filter(file_type=DumpFile.TYPE_CSV, task=task))
def setUpClass(cls): super(ViewTests, cls).setUpClass() es_client = ElasticInstance.get() if es_client.indices.exists(cls.index): es_client.indices.delete(cls.index) es_client.indices.create(cls.index, body=INDEX_CREATION_BODY) cls.catalog_id = 'csv_dump_test_catalog' path = os.path.join(samples_dir, 'distribution_daily_periodicity.json') index_catalog(cls.catalog_id, path, cls.index) cls.task = GenerateDumpTask() cls.task.save() gen = DumpGenerator(cls.task) gen.generate() DumpGenerator(cls.task, cls.catalog_id).generate()
class CSVTest(TestCase): index = 'csv_dump_test_index' # noinspection PyUnresolvedReferences directory = os.path.join(settings.MEDIA_ROOT, 'test_dump') @classmethod def setUpClass(cls): super(CSVTest, cls).setUpClass() cls.catalog_id = 'csv_dump_test_catalog' path = os.path.join(samples_dir, 'distribution_daily_periodicity.json') index_catalog(cls.catalog_id, path, cls.index) cls.task = GenerateDumpTask() cls.task.save() gen = DumpGenerator(cls.task) gen.generate() def test_invalid_catalog(self): task = GenerateDumpTask() task.save() gen = DumpGenerator(task, "no_catalog") gen.generate() task.refresh_from_db() self.assertFalse( DumpFile.objects.filter(file_type=DumpFile.TYPE_CSV, task=task)) def test_values_dump(self): file = self.task.dumpfile_set.get( file_name=DumpFile.FILENAME_VALUES).file reader = read_file_as_csv(file) next(reader) # skip header row = next(reader) self.assertEqual(row[0], self.catalog_id) self.assertEqual(row[6], 'R/P1D') def test_values_length(self): file = self.task.dumpfile_set.get( file_name=DumpFile.FILENAME_VALUES).file reader = read_file_as_csv(file) header = next(reader) self.assertEqual(len(header), 7) def test_entity_identifiers(self): file = self.task.dumpfile_set.get( file_name=DumpFile.FILENAME_VALUES).file reader = read_file_as_csv(file) next(reader) row = next(reader) field_id = row[3] field = Field.objects.get(identifier=field_id) self.assertEqual(self.catalog_id, row[0]) self.assertEqual(field.distribution.identifier, row[2]) self.assertEqual(field.distribution.dataset.identifier, row[1]) self.assertEqual( row[6], field.distribution.enhanced_meta.get( key=meta_keys.PERIODICITY).value) def test_full_csv_zipped(self): dump_file = self.task.dumpfile_set.get( file_name=DumpFile.FILENAME_FULL, file_type=DumpFile.TYPE_CSV) zip_file = ZipDumpFile.objects.get(dump_file=dump_file).file csv_zipped = zipfile.ZipFile(zip_file) full_csv = self.task.dumpfile_set.get(file_name=DumpFile.FILENAME_FULL, file_type=DumpFile.TYPE_CSV) # Necesario para abrir archivos zippeados en modo texto (no bytes) src_file = io.TextIOWrapper(csv_zipped.open(full_csv.get_file_name()), encoding='utf8', newline='') reader = csv.reader(src_file) header = next(reader) self.assertEqual(len(header), 15) def test_values_csv_zipped(self): dump_file = self.task.dumpfile_set.get( file_name=DumpFile.FILENAME_VALUES, file_type=DumpFile.TYPE_CSV) zip_file = ZipDumpFile.objects.get(dump_file=dump_file).file csv_zipped = zipfile.ZipFile(zip_file) # Necesario para abrir archivos zippeados en modo texto (no bytes) src_file = io.TextIOWrapper(csv_zipped.open(dump_file.get_file_name()), encoding='utf8', newline='') reader = csv.reader(src_file) header = next(reader) self.assertEqual(len(header), len(VALUES_HEADER)) def test_full_csv_identifier_fields(self): file = self.task.dumpfile_set.get(file_name=DumpFile.FILENAME_FULL, file_type=DumpFile.TYPE_CSV).file reader = read_file_as_csv(file) next(reader) # Header row = next(reader) field = Field.objects.get(identifier=row[3]) self.assertEqual(row[0], self.catalog_id) self.assertEqual(row[1], field.distribution.dataset.identifier) self.assertEqual(row[2], field.distribution.identifier) self.assertEqual( row[5], field.distribution.enhanced_meta.get( key=meta_keys.PERIODICITY).value) def test_full_csv_metadata_fields(self): file = self.task.dumpfile_set.get(file_name=DumpFile.FILENAME_FULL, file_type=DumpFile.TYPE_CSV).file reader = read_file_as_csv(file) next(reader) # Header row = next(reader) field = Field.objects.get(identifier=row[3]) field_meta = json.loads(field.metadata) distribution_meta = json.loads(field.distribution.metadata) self.assertEqual(row[7], field.title) self.assertEqual(row[8], field_meta['units']) self.assertEqual(row[9], field_meta['description']) self.assertEqual(row[10], distribution_meta['description']) def test_full_csv_dataset_metadata_fields(self): file = self.task.dumpfile_set.get(file_name=DumpFile.FILENAME_FULL, file_type=DumpFile.TYPE_CSV).file reader = read_file_as_csv(file) next(reader) # Header row = next(reader) field = Field.objects.get(identifier=row[3]) dataset_meta = json.loads(field.distribution.dataset.metadata) self.assertEqual(row[12], dataset_meta['publisher']['name']) self.assertEqual(row[13], dataset_meta['source']) self.assertEqual(row[14], field.distribution.dataset.title) def test_full_csv_dataset_theme_field(self): file = self.task.dumpfile_set.get(file_name=DumpFile.FILENAME_FULL, file_type=DumpFile.TYPE_CSV).file reader = read_file_as_csv(file) next(reader) # Header row = next(reader) field = Field.objects.get(identifier=row[3]) dataset_meta = json.loads(field.distribution.dataset.metadata) themes = json.loads( Node.objects.get( catalog_id=self.catalog_id).catalog)['themeTaxonomy'] theme_label = '' for theme in themes: if theme['id'] == dataset_meta['theme'][0]: theme_label = theme['label'] break self.assertEqual(theme_label, row[11]) def test_metadata_csv(self): file = self.task.dumpfile_set.get( file_name=DumpFile.FILENAME_METADATA).file reader = read_file_as_csv(file) next(reader) # self.assertListEqual(header, constants.METADATA_ROWS) self.assertEqual(len(list(reader)), 3) # Un row por serie def test_sources_csv(self): file = self.task.dumpfile_set.get( file_name=DumpFile.FILENAME_SOURCES).file reader = read_file_as_csv(file) next(reader) # Header self.assertEqual(len(list(reader)), 1) # Un row por fuente def test_sources_csv_columns(self): dataset = Field.objects.first().distribution.dataset meta = json.loads(dataset.metadata) file = self.task.dumpfile_set.get( file_name=DumpFile.FILENAME_SOURCES).file reader = read_file_as_csv(file) next(reader) # Header row = next(reader) series = Field.objects.exclude(title='indice_tiempo') self.assertEqual(row[0], meta['source']) # nombre de la fuente self.assertEqual(int(row[1]), 3) # Cantidad de series self.assertEqual( int(row[2]), sum([int(meta_keys.get(x, meta_keys.INDEX_SIZE)) for x in series])) self.assertEqual( row[3], min(meta_keys.get(x, meta_keys.INDEX_START) for x in series)) self.assertEqual( row[4], max(meta_keys.get(x, meta_keys.INDEX_END) for x in series)) def test_leading_nulls_distribution(self): path = os.path.join(samples_dir, 'leading_nulls_distribution.json') index_catalog('leading_null', path, self.index) self.task = GenerateDumpTask() self.task.save() gen = DumpGenerator(self.task, 'leading_null') gen.generate() file = self.task.dumpfile_set.get(file_name=DumpFile.FILENAME_FULL, file_type=DumpFile.TYPE_CSV, node__catalog_id='leading_null').file reader = read_file_as_csv(file) next(reader) # Header!!!! self.assertEqual(len(list(reader)), 1) # Un único row, para un único valor del CSV @classmethod def tearDownClass(cls): super(CSVTest, cls).tearDownClass() ElasticInstance.get().indices.delete(cls.index) Node.objects.all().delete()