def write_metadata_tables(self):
meta = DumpFile.objects.filter(node=self.node,
file_name=DumpFile.FILENAME_METADATA,
file_type=DumpFile.TYPE_CSV).last()
if meta is None or meta.file is None:
return
reader = read_file_as_csv(meta.file)
next(reader) # Skip header
Metadatos.bulk_create(self.generate_series_rows(reader),
batch_size=100)
def generate_series_rows(self, reader):
for row in reader:
catalog_id = row[self.metadata_rows.index(constants.CATALOG_ID)]
dataset_id = row[self.metadata_rows.index(constants.DATASET_ID)]
distribution_id = row[self.metadata_rows.index(
constants.DISTRIBUTION_ID)]
serie_id = row[self.metadata_rows.index(constants.SERIE_ID)]
frequency = row[self.metadata_rows.index(
constants.TIME_INDEX_FREQUENCY)]
title = row[self.metadata_rows.index(constants.SERIES_TITLE)]
units = row[self.metadata_rows.index(constants.SERIES_UNITS)]
description = row[self.metadata_rows.index(
constants.SERIES_DESCRIPTION)]
distribution_title = row[self.metadata_rows.index(
constants.DISTRIBUTION_TITLE)]
distribution_description = row[self.metadata_rows.index(
constants.SERIES_DESCRIPTION)]
distribution_download_url = row[self.metadata_rows.index(
constants.DISTRIBUTION_DOWNLOAD_URL)]
dataset_title = row[self.metadata_rows.index(
constants.DATASET_THEME)]
dataset_description = row[self.metadata_rows.index(
constants.DATASET_DESCRIPTION)]
dataset_source = row[self.metadata_rows.index(
constants.DATASET_SOURCE)]
dataset_publisher = row[self.metadata_rows.index(
constants.DATASET_PUBLISHER)]
dataset_theme = row[self.metadata_rows.index(
constants.DATASET_THEME)]
index_start = row[self.metadata_rows.index(
constants.SERIES_INDEX_START)]
index_end = row[self.metadata_rows.index(
constants.SERIES_INDEX_END)]
value_count = row[self.metadata_rows.index(
constants.SERIES_VALUES_AMT)] or 0
days_not_covered = row[self.metadata_rows.index(
constants.SERIES_DAYS_SINCE_LAST_UPDATE)] or 0
is_updated = bool_format(row[self.metadata_rows.index(
constants.SERIES_IS_UPDATED)])
last_value = row[self.metadata_rows.index(
constants.SERIES_LAST_VALUE)] or 0
second_last_value = row[self.metadata_rows.index(
constants.SERIES_SECOND_LAST_VALUE)] or 0
last_pct_change = row[self.metadata_rows.index(
constants.SERIES_PCT_CHANGE)] or 0
yield Metadatos(
catalogo_id=catalog_id,
dataset_id=dataset_id,
distribucion_id=distribution_id,
serie_id=serie_id,
indice_tiempo_frecuencia=frequency,
serie_titulo=title,
serie_unidades=units,
serie_descripcion=description,
distribucion_titulo=distribution_title,
distribucion_descripcion=distribution_description,
distribucion_url_descarga=distribution_download_url,
dataset_responsable=dataset_publisher,
dataset_fuente=dataset_source,
dataset_titulo=dataset_title,
dataset_descripcion=dataset_description,
dataset_tema=dataset_theme,
serie_indice_inicio=index_start,
serie_indice_final=index_end,
serie_valores_cant=value_count,
serie_dias_no_cubiertos=days_not_covered,
serie_actualizada=is_updated,
serie_valor_ultimo=last_value,
serie_valor_anterior=second_last_value,
serie_var_pct_anterior=last_pct_change,
)
def test_foreign_key(self):
serie = Metadatos.select().first()
values = Valores.filter(serie_id=serie)
self.assertTrue(values)
def test_table_rows(self):
self.assertEqual(Metadatos.select().count(),
Field.objects.exclude(title='indice_tiempo').count())