def test_custom_filter(self): field = self.distribution.field_set.create(title='test_title') field.enhanced_meta.create(key=meta_keys.AVAILABLE, value='true') other_field = self.distribution.field_set.create(title='other_title') other_field.enhanced_meta.create(key=meta_keys.AVAILABLE, value='true') series = SeriesRepository.get_available_series(title='test_title') self.assertEqual(series.count(), 1)
def test_available_and_unavailable_series(self): self.distribution.field_set.create() available_field = self.distribution.field_set.create() available_field.enhanced_meta.create(key=meta_keys.AVAILABLE, value='true') series = SeriesRepository.get_available_series() self.assertEqual(series.count(), 1)
def calculate_distribution_indicators(self, node, data_json, catalog): indexable = Distribution.objects.filter(dataset__catalog=catalog, dataset__indexable=True) updated = indexable.filter(updated=True).count() self.create(type=Indicator.DISTRIBUTION_UPDATED, value=updated, node=node) not_updated = indexable.filter(updated=False).count() self.create(type=Indicator.DISTRIBUTION_NOT_UPDATED, value=not_updated, node=node) discontinued = indexable.filter(present=False).count() self.create(type=Indicator.DISTRIBUTION_INDEXABLE_DISCONTINUED, value=discontinued, node=node) self.create(type=Indicator.DISTRIBUTION_INDEXABLE, value=indexable.count(), node=node) not_indexable = Distribution.objects.filter(dataset__catalog=catalog, dataset__indexable=False) new = not_indexable.filter(new=True).count() self.create(type=Indicator.DISTRIBUTION_NEW, value=new, node=node) previous = not_indexable.filter(new=False, present=True).count() self.create(type=Indicator.DISTRIBUTION_NOT_INDEXABLE_PREVIOUS, value=previous, node=node) error = Distribution.objects.filter(dataset__catalog=catalog, error=True).count() self.create(type=Indicator.DISTRIBUTION_ERROR, value=error, node=node) not_indexable_discontinued = not_indexable.filter( present=False).count() self.create(type=Indicator.DISTRIBUTION_NOT_INDEXABLE_DISCONTINUED, value=not_indexable_discontinued, node=node) self.create(type=Indicator.DISTRIBUTION_NOT_INDEXABLE, value=not_indexable.count(), node=node) self.create(type=Indicator.DISTRIBUTION_AVAILABLE, value=len( data_json.get_distributions(only_time_series=True)), node=node) available_series = SeriesRepository.get_available_series() self.create(type=Indicator.DISTRIBUTION_TOTAL, value=available_series.filter( distribution__dataset__catalog=catalog).values_list( 'distribution').distinct().count(), node=node)
def get_all_available_series_units() -> set: series_metadata = SeriesRepository.get_available_series() \ .values_list('metadata', flat=True) catalog_units = set() for metadata in series_metadata: metadata = json.loads(metadata) units = metadata.get('units') if units: catalog_units.add(units) return catalog_units
def init_data(self): """Inicializa en un diccionario con IDs de series como clave los valores a escribir en cada uno de los CSV. """ fields = SeriesRepository.get_available_series().exclude(identifier=None) if self.catalog: try: catalog = Catalog.objects.get(identifier=self.catalog) except Catalog.DoesNotExist: return fields = fields.filter( distribution__dataset__catalog=catalog ) fields = fields.prefetch_related( 'distribution', 'distribution__dataset', 'distribution__dataset__catalog', 'enhanced_meta', ) all_meta = Metadata.objects.all() field_ct = ContentType.objects.get_for_model(Field) for field in fields: meta = json.loads(field.metadata) dist_meta = json.loads(field.distribution.metadata) dataset_meta = json.loads(field.distribution.dataset.metadata) themes = field.distribution.dataset.themes theme_labels = get_theme_labels(json.loads(themes)) if themes else '' self.fields[field.identifier] = { 'dataset': field.distribution.dataset, 'distribution': field.distribution, 'serie': field, 'serie_titulo': field.title, 'serie_unidades': meta.get('units'), 'serie_descripcion': meta.get('description'), 'distribucion_titulo': dist_meta.get('title'), 'distribucion_descripcion': dist_meta.get('description'), 'distribucion_url_descarga': field.distribution.download_url, 'dataset_responsable': dataset_meta.get('publisher', {}).get('name'), 'dataset_fuente': dataset_meta.get('source'), 'dataset_titulo': field.distribution.dataset.title, 'dataset_descripcion': dataset_meta.get('description'), 'dataset_tema': theme_labels, 'metadata': {o.key: o.value for o in list(all_meta.filter(content_type=field_ct, object_id=field.id))}, 'frequency': self.serie_periodicity(field), }
def _get_model(self, series_id): """Valida si el 'series_id' es válido, es decir, si la serie pedida es un ID contenido en la base de datos. De no encontrarse, llena la lista de errores según corresponda. """ field_model = SeriesRepository.get_available_series(identifier=series_id).first() if field_model is None: self._append_error(SERIES_DOES_NOT_EXIST.format(series_id), series_id=series_id) return None index_start_metadata = meta_keys.get(field_model, meta_keys.INDEX_START) if index_start_metadata is None: self._append_error(SERIES_DOES_NOT_EXIST.format(series_id), series_id=series_id) return None return field_model
def update_popularity_metadata(distribution: Distribution): if not Index(SeriesQuery._doc_type.index).exists(): return series = SeriesRepository.get_available_series(distribution=distribution) series_ids = series.values_list('identifier', flat=True) if not series_ids: return for meta_key, days in KEY_DAYS_PAIRS: s = SeriesQuery.search() if days: s = s.filter('range', timestamp={'gte': f'now-{days}d/d'}) buckets = { serie_id: get_serie_filter(serie_id) for serie_id in series_ids } agg_result = popularity_aggregation(s, buckets) update_series_popularity_metadata(agg_result, meta_key, series)
def get_available_fields(self): return SeriesRepository.get_available_series( distribution__dataset__catalog__identifier=self.node.catalog_id )
def all_time_series(): series_ids = SeriesRepository.get_available_series()\ .values_list('identifier', flat=True) return series_ids
def test_non_present_series(self): non_present_field = self.distribution.field_set.create(present=False) non_present_field.enhanced_meta.create(key=meta_keys.AVAILABLE, value='true') series = SeriesRepository.get_available_series() self.assertTrue(series)
def test_one_available_series_shows_up_in_result(self): field = self.distribution.field_set.create() field.enhanced_meta.create(key=meta_keys.AVAILABLE, value='true') series = SeriesRepository.get_available_series() self.assertTrue(series)
def test_no_available_series_no_results(self): self.distribution.field_set.create() series = SeriesRepository.get_available_series() self.assertFalse(series)
def test_no_series_no_results(self): series = SeriesRepository.get_available_series() self.assertFalse(series)