def parse_activities(self, root):
"""
"""
for e in root.getchildren():
self.model_store = OrderedDict()
parsed = self.parse(e)
# only save if the activity is updated
if parsed:
try:
self.save_all_models()
self.post_save_models()
except Exception:
model = self.get_model('Activity')
if model is not None:
ActivityTaskIndexing(model, related=True).run()
else:
model = self.get_model('Activity')
if model is not None:
ActivityTaskIndexing(model, related=True).run()
self.post_save_file(self.dataset)
if settings.ERROR_LOGS_ENABLED:
self.post_save_validators(self.dataset)
# TODO - only delete errors on activities that were updated
self.dataset.note_count = len(self.errors)
self.dataset.save()
DatasetNote.objects.filter(dataset=self.dataset).delete()
DatasetNote.objects.bulk_create(self.errors)
DatasetNoteTaskIndexing().run_from_dataset(dataset=self.dataset)
def add_dataset_note_to_solr(dataset_note_id):
try:
DatasetNoteTaskIndexing(
instance=DatasetNote.objects.get(id=dataset_note_id),
related=True).run()
except DatasetNote.DoesNotExist:
pass
def dataset_note_post_save(sender, instance, **kwargs):
DatasetNoteTaskIndexing(instance=instance).run()
def dataset_note_pre_delete(sender, instance, **kwargs):
DatasetNoteTaskIndexing(instance=instance).delete()
def parse_activities(self, root):
"""
"""
activities_to_keep = []
for e in root.getchildren():
self.model_store = OrderedDict()
parsed = False
should_be_parsed = True
hasher = hashlib.sha1()
hasher.update(etree.tostring(e))
sha1 = hasher.hexdigest()
if not self.force_reparse:
try:
iati_id = e.find('iati-identifier').text
saved_activity = Activity.objects.\
filter(iati_identifier=iati_id).values("sha1")
if saved_activity.count() == 1 \
and saved_activity[0]['sha1'] == sha1:
activities_to_keep.append(iati_id)
should_be_parsed = False
except Exception:
should_be_parsed = True
if should_be_parsed:
parsed = self.parse(e)
# only save if the activity is updated
if parsed:
try:
model = self.get_model('Activity')
if model is not None and sha1:
model.sha1 = sha1
self.save_all_models()
self.post_save_models()
except Exception:
model = self.get_model('Activity')
if model is not None and sha1:
model.sha1 = sha1
ActivityTaskIndexing(model,
related=True).run()
else:
model = self.get_model('Activity')
if model is not None and sha1:
model.sha1 = sha1
ActivityTaskIndexing(model, related=True).run()
self.post_save_file(self.dataset, activities_to_keep)
if settings.ERROR_LOGS_ENABLED:
self.post_save_validators(self.dataset)
# TODO - only delete errors on activities that were updated
self.dataset.note_count = len(self.errors)
self.dataset.save()
DatasetNote.objects.filter(dataset=self.dataset).delete()
DatasetNote.objects.bulk_create(self.errors)
DatasetNoteTaskIndexing().run_from_dataset(dataset=self.dataset)