def write_metadata_tables(self): meta = DumpFile.objects.filter(node=self.node, file_name=DumpFile.FILENAME_METADATA, file_type=DumpFile.TYPE_CSV).last() if meta is None or meta.file is None: return reader = read_file_as_csv(meta.file) next(reader) # Skip header Metadatos.bulk_create(self.generate_series_rows(reader), batch_size=100)
def generate_series_rows(self, reader): for row in reader: catalog_id = row[self.metadata_rows.index(constants.CATALOG_ID)] dataset_id = row[self.metadata_rows.index(constants.DATASET_ID)] distribution_id = row[self.metadata_rows.index( constants.DISTRIBUTION_ID)] serie_id = row[self.metadata_rows.index(constants.SERIE_ID)] frequency = row[self.metadata_rows.index( constants.TIME_INDEX_FREQUENCY)] title = row[self.metadata_rows.index(constants.SERIES_TITLE)] units = row[self.metadata_rows.index(constants.SERIES_UNITS)] description = row[self.metadata_rows.index( constants.SERIES_DESCRIPTION)] distribution_title = row[self.metadata_rows.index( constants.DISTRIBUTION_TITLE)] distribution_description = row[self.metadata_rows.index( constants.SERIES_DESCRIPTION)] distribution_download_url = row[self.metadata_rows.index( constants.DISTRIBUTION_DOWNLOAD_URL)] dataset_title = row[self.metadata_rows.index( constants.DATASET_THEME)] dataset_description = row[self.metadata_rows.index( constants.DATASET_DESCRIPTION)] dataset_source = row[self.metadata_rows.index( constants.DATASET_SOURCE)] dataset_publisher = row[self.metadata_rows.index( constants.DATASET_PUBLISHER)] dataset_theme = row[self.metadata_rows.index( constants.DATASET_THEME)] index_start = row[self.metadata_rows.index( constants.SERIES_INDEX_START)] index_end = row[self.metadata_rows.index( constants.SERIES_INDEX_END)] value_count = row[self.metadata_rows.index( constants.SERIES_VALUES_AMT)] or 0 days_not_covered = row[self.metadata_rows.index( constants.SERIES_DAYS_SINCE_LAST_UPDATE)] or 0 is_updated = bool_format(row[self.metadata_rows.index( constants.SERIES_IS_UPDATED)]) last_value = row[self.metadata_rows.index( constants.SERIES_LAST_VALUE)] or 0 second_last_value = row[self.metadata_rows.index( constants.SERIES_SECOND_LAST_VALUE)] or 0 last_pct_change = row[self.metadata_rows.index( constants.SERIES_PCT_CHANGE)] or 0 yield Metadatos( catalogo_id=catalog_id, dataset_id=dataset_id, distribucion_id=distribution_id, serie_id=serie_id, indice_tiempo_frecuencia=frequency, serie_titulo=title, serie_unidades=units, serie_descripcion=description, distribucion_titulo=distribution_title, distribucion_descripcion=distribution_description, distribucion_url_descarga=distribution_download_url, dataset_responsable=dataset_publisher, dataset_fuente=dataset_source, dataset_titulo=dataset_title, dataset_descripcion=dataset_description, dataset_tema=dataset_theme, serie_indice_inicio=index_start, serie_indice_final=index_end, serie_valores_cant=value_count, serie_dias_no_cubiertos=days_not_covered, serie_actualizada=is_updated, serie_valor_ultimo=last_value, serie_valor_anterior=second_last_value, serie_var_pct_anterior=last_pct_change, )
def test_foreign_key(self): serie = Metadatos.select().first() values = Valores.filter(serie_id=serie) self.assertTrue(values)
def test_table_rows(self): self.assertEqual(Metadatos.select().count(), Field.objects.exclude(title='indice_tiempo').count())