def update_distribution_metadata(changed, distribution_model):
time_index = DistributionRepository(
distribution_model).get_time_index_series()
df = init_df(distribution_model, time_index)
periodicity = get_distribution_time_index_periodicity(time_index)
new_metadata = []
metas_to_delete = []
field_content_type = ContentType.objects.get_for_model(Field)
for serie in list(df.columns):
meta = calculate_enhanced_meta(df[serie], periodicity)
field = distribution_model.field_set.get(identifier=serie,
present=True)
for meta_key, value in meta.items():
new_metadata.append(
Metadata(content_type=field_content_type,
object_id=field.id,
key=meta_key,
value=value))
metas_to_delete.extend(
Metadata.objects.filter(object_id=field.id,
key__in=list(meta.keys())).values_list(
'id', flat=True))
with transaction.atomic():
Metadata.objects.filter(id__in=metas_to_delete).delete()
Metadata.objects.bulk_create(new_metadata)
distribution_model.enhanced_meta.update_or_create(
key=meta_keys.LAST_HASH,
defaults={'value': distribution_model.data_hash})
distribution_model.enhanced_meta.update_or_create(
key=meta_keys.CHANGED, defaults={'value': str(changed)})
update_popularity_metadata(distribution_model)
remove_duplicated_fields(distribution_model)
def test_is_updated(self):
with freeze_time(self.df.index[-1]):
meta = calculate_enhanced_meta(self.df[self.df.columns[0]], self.periodicity)
self.assertTrue(meta.get(meta_keys.IS_UPDATED))
def test_is_not_updated(self):
meta = calculate_enhanced_meta(self.df[self.df.columns[0]], self.periodicity)
self.assertFalse(meta.get(meta_keys.IS_UPDATED))
def setUp(self):
self.periodicity = 'R/P1D'
path = os.path.join(SAMPLES_DIR, 'daily_periodicity.csv')
self.df = pd.read_csv(path, parse_dates=['indice_tiempo'], index_col='indice_tiempo')
self.meta = calculate_enhanced_meta(self.df[self.df.columns[0]], self.periodicity)
def test_second_to_last_value_uses_last_valid_index(self):
path = os.path.join(SAMPLES_DIR, 'daily_periodicity_missing_last_value.csv')
df = pd.read_csv(path, parse_dates=['indice_tiempo'], index_col='indice_tiempo')
meta = calculate_enhanced_meta(df[df.columns[0]], self.periodicity)
self.assertEqual(5.3504, meta.get(meta_keys.SECOND_TO_LAST_VALUE))
def test_last_value_takes_last_non_nan_value(self):
path = os.path.join(SAMPLES_DIR, 'daily_periodicity_missing_last_value.csv')
df = pd.read_csv(path, parse_dates=['indice_tiempo'], index_col='indice_tiempo')
meta = calculate_enhanced_meta(df[df.columns[0]], self.periodicity)
self.assertEqual(5.0931, meta.get(meta_keys.LAST_VALUE))