def test_val_diff(self):
self.assertEqual(dict_diff({}, {"un": 1}), {"un": {"added": 1}})
self.assertEqual(dict_diff({"un": 1}, {}), {"un": {"removed": 1}})
self.assertEqual(dict_diff({"un": 1}, {"un": 2}),
{"un": {
"removed": 1,
"added": 2
}})
def test_list_diff(self):
self.assertEqual(dict_diff({}, {"un": [1]}), {"un": {"added": [1]}})
self.assertEqual(dict_diff({"un": [1]}, {}), {"un": {"removed": [1]}})
self.assertEqual(dict_diff({"un": [1]}, {"un": [2]}),
{"un": {
"removed": [1],
"added": [2]
}})
def test_list_diff(self):
self.assertEqual(dict_diff({}, {"un": [1]}),
{"un": {"added": [1]}})
self.assertEqual(dict_diff({"un": [1]}, {}),
{"un": {"removed": [1]}})
self.assertEqual(dict_diff({"un": [1]}, {"un": [2]}),
{"un": {"removed": [1],
"added": [2]}})
def test_all_in_one(self):
self.assertEqual(
dict_diff({
"un": 1,
"list": [{
"un": 1
}, {
"deux": 2
}],
"str": "ha!"
}, {
"list": [{
"deux": 2
}, {
"trois": 3
}],
"str": "ha!"
}), {
"un": {
"removed": 1
},
"list": {
"removed": [{
"un": 1
}],
"added": [{
"trois": 3
}]
}
})
def test_all_in_one(self):
self.assertEqual(dict_diff({"un": 1,
"list": [{"un": 1}, {"deux": 2}],
"str": "ha!"},
{"list": [{"deux": 2}, {"trois": 3}],
"str": "ha!"}),
{"un": {"removed": 1},
"list": {"removed": [{"un": 1}],
"added": [{"trois": 3}]}})
def sync_feed(feed):
now = timezone.now()
# feed
feed_updated = False
if feed.url:
feed_serializer = get_feed_serializer(feed.url)
elif feed.path:
feed_path = Feed._meta.get_field('path').path
full_path = os.path.join(feed_path, feed.path)
feed_serializer = get_feed_serializer_for_path(full_path)
current_feed_name = feed_serializer.get_name(feed.url)
if not feed.name == current_feed_name:
feed_updated = True
feed.name = current_feed_name
current_feed_description = feed_serializer.validated_data.get(
"description") or ""
if not feed.description == current_feed_description:
feed_updated = True
feed.description = current_feed_description
# feed probes
seen_keys = []
created = updated = archived = removed = 0
# created / updated
feed_probes = list(feed_serializer.iter_feed_probes()
) # too trigger the FeedErrors before touching the DB
for feed_probe_key, feed_probe_data in feed_probes:
seen_keys.append(feed_probe_key)
feed_probe, fp_created = FeedProbe.objects.get_or_create(
feed=feed, key=feed_probe_key, defaults=feed_probe_data)
if not fp_created:
if feed_probe.model != feed_probe_data["model"]:
raise FeedError(
"Can't change feed probe {} model.".format(feed_probe_key))
feed_probe_data["archived_at"] = None
diff = dict_diff(
{
"model": feed_probe.model,
"name": feed_probe.name,
"description": feed_probe.description,
"body": feed_probe.body,
"archived_at": feed_probe.archived_at
}, feed_probe_data)
if diff:
for key, val in feed_probe_data.items():
setattr(feed_probe, key, val)
feed_probe.save()
# mark all imported probe for update review
for probe_source in feed_probe.probesource_set.all():
if probe_source.update_diff():
if not probe_source.feed_probe_update_available:
probe_source.feed_probe_update_available = True
probe_source.save()
elif probe_source.feed_probe_update_available:
probe_source.feed_probe_update_available = False
probe_source.save()
updated += 1
else:
created += 1
# feed_probes not in feed
feed_probe_not_in_feed_qs = feed.feedprobe_set.exclude(key__in=seen_keys)
# archive stale feed probes linked to probe_sources
for feed_probe in feed_probe_not_in_feed_qs.filter(
probesource__isnull=False, archived_at__isnull=True):
feed_probe.archived_at = now
feed_probe.save()
archived += 1
# remove stale feed probes not linked to any probe_source
for feed_probe in feed_probe_not_in_feed_qs.filter(
probesource__isnull=True):
feed_probe.delete()
removed += 1
operations = OrderedDict(
(l, v) for l, v in (("created", created), ("updated", updated),
("archived", archived), ("removed", removed)) if v)
feed.last_synced_at = now
if feed_updated or operations:
feed.updated_at = now
feed.save()
return operations
def update_diff(self):
if not self.feed_probe:
return {}
probe = self.load()
current_body = probe.export()["body"]
return dict_diff(current_body, self.feed_probe.body)
def sync_feed(feed):
now = timezone.now()
# feed
feed_updated = False
feed_serializer = get_feed_serializer(feed.url)
current_feed_name = feed_serializer.get_name(feed.url)
if not feed.name == current_feed_name:
feed_updated = True
feed.name = current_feed_name
current_feed_description = feed_serializer.validated_data.get("description") or ""
if not feed.description == current_feed_description:
feed_updated = True
feed.description = current_feed_description
# feed probes
seen_keys = []
created = updated = archived = removed = 0
# created / updated
feed_probes = list(feed_serializer.iter_feed_probes()) # too trigger the FeedErrors before touching the DB
for feed_probe_key, feed_probe_data in feed_probes:
seen_keys.append(feed_probe_key)
feed_probe, fp_created = FeedProbe.objects.get_or_create(feed=feed, key=feed_probe_key,
defaults=feed_probe_data)
if not fp_created:
if feed_probe.model != feed_probe_data["model"]:
raise FeedError("Can't change feed probe {} model.".format(feed_probe_key))
feed_probe_data["archived_at"] = None
diff = dict_diff({"model": feed_probe.model,
"name": feed_probe.name,
"description": feed_probe.description,
"body": feed_probe.body,
"archived_at": feed_probe.archived_at},
feed_probe_data)
if diff:
for key, val in feed_probe_data.items():
setattr(feed_probe, key, val)
feed_probe.save()
# mark all imported probe for update review
for probe_source in feed_probe.probesource_set.all():
if probe_source.update_diff():
if not probe_source.feed_probe_update_available:
probe_source.feed_probe_update_available = True
probe_source.save()
elif probe_source.feed_probe_update_available:
probe_source.feed_probe_update_available = False
probe_source.save()
updated += 1
else:
created += 1
# feed_probes not in feed
feed_probe_not_in_feed_qs = feed.feedprobe_set.exclude(key__in=seen_keys)
# archive stale feed probes linked to probe_sources
for feed_probe in feed_probe_not_in_feed_qs.filter(probesource__isnull=False,
archived_at__isnull=True):
feed_probe.archived_at = now
feed_probe.save()
archived += 1
# remove stale feed probes not linked to any probe_source
for feed_probe in feed_probe_not_in_feed_qs.filter(probesource__isnull=True):
feed_probe.delete()
removed += 1
operations = OrderedDict((l, v)
for l, v in (("created", created),
("updated", updated),
("archived", archived),
("removed", removed))
if v)
feed.last_synced_at = now
if feed_updated or operations:
feed.updated_at = now
feed.save()
return operations
def update_diff(self):
if not self.feed_probe:
return {}
probe = self.load()
current_body = probe.export()["body"]
return dict_diff(current_body, self.feed_probe.body)
def test_none(self):
self.assertEqual(dict_diff({}, {"archived_at": None}), {})
def test_empty_diff(self):
self.assertEqual(dict_diff({}, {}), {})
def test_none(self):
self.assertEqual(dict_diff({}, {"archived_at": None}), {})
def test_empty_diff(self):
self.assertEqual(dict_diff({}, {}), {})
def test_val_diff(self):
self.assertEqual(dict_diff({}, {"un": 1}), {"un": {"added": 1}})
self.assertEqual(dict_diff({"un": 1}, {}), {"un": {"removed": 1}})
self.assertEqual(dict_diff({"un": 1}, {"un": 2}),
{"un": {"removed": 1,
"added": 2}})