Ejemplo n.º 1
0
    def parse_activities(self, root):
        """

        """
        for e in root.getchildren():
            self.model_store = OrderedDict()
            parsed = self.parse(e)
            # only save if the activity is updated

            if parsed:
                try:
                    self.save_all_models()
                    self.post_save_models()
                except Exception:
                    model = self.get_model('Activity')
                    if model is not None:
                        ActivityTaskIndexing(model, related=True).run()
                else:
                    model = self.get_model('Activity')
                    if model is not None:
                        ActivityTaskIndexing(model, related=True).run()

        self.post_save_file(self.dataset)

        if settings.ERROR_LOGS_ENABLED:
            self.post_save_validators(self.dataset)

            # TODO - only delete errors on activities that were updated
            self.dataset.note_count = len(self.errors)
            self.dataset.save()

            DatasetNote.objects.filter(dataset=self.dataset).delete()
            DatasetNote.objects.bulk_create(self.errors)

            DatasetNoteTaskIndexing().run_from_dataset(dataset=self.dataset)
Ejemplo n.º 2
0
def add_dataset_note_to_solr(dataset_note_id):
    try:
        DatasetNoteTaskIndexing(
            instance=DatasetNote.objects.get(id=dataset_note_id),
            related=True).run()
    except DatasetNote.DoesNotExist:
        pass
Ejemplo n.º 3
0
def dataset_note_post_save(sender, instance, **kwargs):
    DatasetNoteTaskIndexing(instance=instance).run()
Ejemplo n.º 4
0
def dataset_note_pre_delete(sender, instance, **kwargs):
    DatasetNoteTaskIndexing(instance=instance).delete()
Ejemplo n.º 5
0
    def parse_activities(self, root):
        """

        """
        activities_to_keep = []
        for e in root.getchildren():
            self.model_store = OrderedDict()
            parsed = False
            should_be_parsed = True

            hasher = hashlib.sha1()
            hasher.update(etree.tostring(e))
            sha1 = hasher.hexdigest()
            if not self.force_reparse:
                try:
                    iati_id = e.find('iati-identifier').text
                    saved_activity = Activity.objects.\
                        filter(iati_identifier=iati_id).values("sha1")
                    if saved_activity.count() == 1 \
                            and saved_activity[0]['sha1'] == sha1:
                        activities_to_keep.append(iati_id)
                        should_be_parsed = False
                except Exception:
                    should_be_parsed = True

            if should_be_parsed:
                parsed = self.parse(e)
            # only save if the activity is updated

            if parsed:
                try:
                    model = self.get_model('Activity')
                    if model is not None and sha1:
                        model.sha1 = sha1
                    self.save_all_models()
                    self.post_save_models()
                except Exception:
                    model = self.get_model('Activity')
                    if model is not None and sha1:
                        model.sha1 = sha1
                        ActivityTaskIndexing(model,
                                             related=True).run()
                else:
                    model = self.get_model('Activity')
                    if model is not None and sha1:
                        model.sha1 = sha1
                        ActivityTaskIndexing(model, related=True).run()

        self.post_save_file(self.dataset, activities_to_keep)

        if settings.ERROR_LOGS_ENABLED:
            self.post_save_validators(self.dataset)

            # TODO - only delete errors on activities that were updated
            self.dataset.note_count = len(self.errors)
            self.dataset.save()

            DatasetNote.objects.filter(dataset=self.dataset).delete()
            DatasetNote.objects.bulk_create(self.errors)

            DatasetNoteTaskIndexing().run_from_dataset(dataset=self.dataset)