def update_distribution_metadata(changed, distribution_model): time_index = DistributionRepository( distribution_model).get_time_index_series() df = init_df(distribution_model, time_index) periodicity = get_distribution_time_index_periodicity(time_index) new_metadata = [] metas_to_delete = [] field_content_type = ContentType.objects.get_for_model(Field) for serie in list(df.columns): meta = calculate_enhanced_meta(df[serie], periodicity) field = distribution_model.field_set.get(identifier=serie, present=True) for meta_key, value in meta.items(): new_metadata.append( Metadata(content_type=field_content_type, object_id=field.id, key=meta_key, value=value)) metas_to_delete.extend( Metadata.objects.filter(object_id=field.id, key__in=list(meta.keys())).values_list( 'id', flat=True)) with transaction.atomic(): Metadata.objects.filter(id__in=metas_to_delete).delete() Metadata.objects.bulk_create(new_metadata) distribution_model.enhanced_meta.update_or_create( key=meta_keys.LAST_HASH, defaults={'value': distribution_model.data_hash}) distribution_model.enhanced_meta.update_or_create( key=meta_keys.CHANGED, defaults={'value': str(changed)}) update_popularity_metadata(distribution_model) remove_duplicated_fields(distribution_model)
def test_is_updated(self): with freeze_time(self.df.index[-1]): meta = calculate_enhanced_meta(self.df[self.df.columns[0]], self.periodicity) self.assertTrue(meta.get(meta_keys.IS_UPDATED))
def test_is_not_updated(self): meta = calculate_enhanced_meta(self.df[self.df.columns[0]], self.periodicity) self.assertFalse(meta.get(meta_keys.IS_UPDATED))
def setUp(self): self.periodicity = 'R/P1D' path = os.path.join(SAMPLES_DIR, 'daily_periodicity.csv') self.df = pd.read_csv(path, parse_dates=['indice_tiempo'], index_col='indice_tiempo') self.meta = calculate_enhanced_meta(self.df[self.df.columns[0]], self.periodicity)
def test_second_to_last_value_uses_last_valid_index(self): path = os.path.join(SAMPLES_DIR, 'daily_periodicity_missing_last_value.csv') df = pd.read_csv(path, parse_dates=['indice_tiempo'], index_col='indice_tiempo') meta = calculate_enhanced_meta(df[df.columns[0]], self.periodicity) self.assertEqual(5.3504, meta.get(meta_keys.SECOND_TO_LAST_VALUE))
def test_last_value_takes_last_non_nan_value(self): path = os.path.join(SAMPLES_DIR, 'daily_periodicity_missing_last_value.csv') df = pd.read_csv(path, parse_dates=['indice_tiempo'], index_col='indice_tiempo') meta = calculate_enhanced_meta(df[df.columns[0]], self.periodicity) self.assertEqual(5.0931, meta.get(meta_keys.LAST_VALUE))