Exemple #1
0
 def _publish(self):
     has_image = self.project.current_image != ''
     has_lat_long = self.project.primary_location.latitude != 0 or self.project.primary_location.longitude != 0
     if has_image and has_lat_long:
         self.project.publishingstatus.status = PublishingStatus.STATUS_PUBLISHED
         log(
            "Project ID: {rsr_id} published",
            dict(
                rsr_id=self.project.pk,
                internal_id=self.activity.internal_id(),
                iati_id=self.activity.iati_id(),
                event=ACTION_PROJECT_PUBLISHED,
            )
         )
     else:
         self.project.publishingstatus.status = PublishingStatus.STATUS_UNPUBLISHED
         log(
            "Project ID: {rsr_id} not published",
            dict(
                rsr_id=self.project.pk,
                internal_id=self.activity.internal_id(),
                iati_id=self.activity.iati_id(),
                event=ACTION_PROJECT_NOT_PUBLISHED,
            )
         )
     self.project.publishingstatus.save()
def upload_activities(argv):
    user = credentials_from_args(argv)
    if user:
        xml = load_xml(RAIN_IATI_ACTIVITES_URL)
        if xml:
            save_xml(xml, "rain_activities_{datetime}.xml")
            parser = etree.XMLParser(ns_clean=True, recover=True, encoding="utf-8")
            root = etree.fromstring(xml, parser=parser)
            document_akvo_ns = root.nsmap["akvo"]
            assert document_akvo_ns == AKVO_NS, "Akvo name space is incorrect in the IATI XML"
            activities = root.findall("iati-activity")
            activity_count = len(activities)
            for i, activity in enumerate(activities):
                activity = RainActivity(activity, RAIN_ACTIVITY_NS, AKVO_NS)
                internal_id = activity.internal_id()
                iati_id = activity.iati_id()
                try:
                    assert iati_id is not None, "No IATI ID found, for activity number {} in the XML".format(i + 1)
                except AssertionError, e:
                    message = "No IATI ID for activity number {extra}"
                    data = dict(event=ERROR_MISSING_IATI_ID, extra=i)
                    log(message, data)
                    print message.format(**data)
                    continue
                rsr_id = activity.rsr_id()
                print "({current} of {activity_count}) Processing activity {iati_id}".format(
                    current=i + 1, activity_count=activity_count, iati_id=iati_id
                ),
                if len(activity.tree.findall("participating-org")) > 0:
                    try:
                        rsr_id = identify_rsr_project(user, rsr_id, iati_id, internal_id)
                    except AssertionError, e:
                        message = "Error identifying RSR project: ID {pk}, IATI ID: {iati_id}, Error message: \n{extra}"
                        data = dict(pk=rsr_id, iati_id=iati_id, event=ERROR_IDENTIFY_RSR_PROJECT, extra=e.message)
                        log(message, data)
                        print message.format(**data)
                        continue
                    # HACK! removing duplicates of RAIN as participating-org
                    for dupe_rain in activity.tree.xpath(
                        'participating-org[@akvo:ref="0"]', namespaces={"akvo": "http://akvo.org/iati-activities"}
                    ):
                        dupe_rain.getparent().remove(dupe_rain)
                    if rsr_id:
                        ok, message, data = put_an_activity(activity.tree, rsr_id, user)
                        log(message, data)
                        print message.format(**data)
                    else:
                        ok, message, data = post_an_activity(activity.tree, user)
                        log(message, data)
                        print message.format(**data)
                else:
                    message = "Iati-activity {iati_id} has no participating-orgs, aborting"
                    data = dict(iati_id=iati_id, event=ERROR_NO_ORGS)
                    log(message, data)
                    print (message.format(**data))
Exemple #3
0
 def _current_image(self):
     image = ImageImporter(self.activity.current_image())
     try:
         image.get_image()
     except Exception, e:
         log(
             "Error trying to fetch image to project. Image URL: {extra}",
             dict(rsr_id=self.activity.rsr_id(),
                  internal_id=self.activity.internal_id(),
                  iati_id=self.activity.iati_id(),
                  event=ERROR_IMAGE_UPLOAD,
                  extra=self.activity.current_image()))
Exemple #4
0
 def _process_project(self, activity):
     try:
         project_saver = ProjectSaver(activity, self.importing_org)
     except Project.DoesNotExist, e:
         log(
             "Couldn't find project ID: {rsr_id} IATI ID: {iati_id} internal ID: {internal_id}",
             dict(
                 internal_id=activity.internal_id(),
                 rsr_id=activity.rsr_id(),
                 iati_id=activity.iati_id(),
                 event=ERROR_PROJECT_NOT_FOUND,
             ))
         return None
Exemple #5
0
 def _process_project(self, activity):
     try:
         project_saver = ProjectSaver(activity, self.importing_org)
     except Project.DoesNotExist, e:
         log(
            "Couldn't find project ID: {rsr_id} IATI ID: {iati_id} internal ID: {internal_id}",
            dict(
                internal_id=activity.internal_id(),
                rsr_id=activity.rsr_id(),
                iati_id=activity.iati_id(),
                event=ERROR_PROJECT_NOT_FOUND,
            )
         )
         return None
Exemple #6
0
 def __init__(self, activity, importing_org):
     self.activity = activity
     if activity.rsr_id():
         try:
             self.project = Project.objects.get(id=activity.rsr_id())
             return
         except Project.DoesNotExist, e:
             msg = "Could not find project with ID: {rsr_id}"
             log(
                 msg,
                 dict(rsr_id=activity.rsr_id(),
                      internal_id=activity.internal_id(),
                      iati_id=activity.iati_id(),
                      event=ERROR_IMAGE_UPLOAD,
                      extra=msg.format(rsr_id=activity.rsr_id())))
Exemple #7
0
 def _current_image(self):
     image = ImageImporter(self.activity.current_image())
     try:
         image.get_image()
     except Exception, e:
         log(
             "Error trying to fetch image to project. Image URL: {extra}",
             dict(
                 rsr_id=self.activity.rsr_id(),
                 internal_id=self.activity.internal_id(),
                 iati_id=self.activity.iati_id(),
                 event=ERROR_IMAGE_UPLOAD,
                 extra=self.activity.current_image()
             )
         )
Exemple #8
0
class ProjectSaver():
    """ Perform the post import saving steps for the project, all related to the project image
    """
    def __init__(self, activity, importing_org):
        self.activity = activity
        if activity.rsr_id():
            try:
                self.project = Project.objects.get(id=activity.rsr_id())
                return
            except Project.DoesNotExist, e:
                msg = "Could not find project with ID: {rsr_id}"
                log(
                    msg,
                    dict(rsr_id=activity.rsr_id(),
                         internal_id=activity.internal_id(),
                         iati_id=activity.iati_id(),
                         event=ERROR_IMAGE_UPLOAD,
                         extra=msg.format(rsr_id=activity.rsr_id())))
        elif activity.internal_id():
            try:
                self.project = Project.objects.get(
                    partnerships__internal_id=activity.internal_id(),
                    partnerships__organisation=RAIN_ORG_ID)
                return
            except:
                msg = "Could not find project or multiple projects found with RAIN internal ID: {internal_id}"
                log(
                    msg,
                    dict(rsr_id=self.activity.rsr_id(),
                         internal_id=self.activity.internal_id(),
                         iati_id=self.activity.iati_id(),
                         event=ERROR_IMAGE_UPLOAD,
                         extra=msg.format(internal_id=activity.internal_id())))
Exemple #9
0
 def __init__(self, activity, importing_org):
     self.activity = activity
     if activity.rsr_id():
         try:
             self.project = Project.objects.get(id=activity.rsr_id())
             return
         except Project.DoesNotExist, e:
             msg = "Could not find project with ID: {rsr_id}"
             log(
                 msg,
                 dict(
                     rsr_id=activity.rsr_id(),
                     event=ERROR_IMAGE_UPLOAD,
                     extra=msg.format(rsr_id=activity.rsr_id())
                 )
             )
Exemple #10
0
 def process(self):
     self._current_image()
     self._current_image_caption()
     self._current_image_credit()
     try:
         self.project.full_clean()
     except ValidationError, e:
         log(
            "Data validation error when saving project ID: {rsr_id}:\n{extra}",
            dict(
                rsr_id=self.project.pk,
                event=ERROR_PROJECT_DATA_INVALID,
                extra=e,
            )
         )
         return
Exemple #11
0
 def process(self):
     self._current_image()
     self._current_image_caption()
     self._current_image_credit()
     self._category()
     self._sync_owner()
     self._publish()
     try:
         self.project.full_clean()
     except ValidationError, e:
         log(
             "Warning: data validation error when saving project ID: {rsr_id}:\n{extra}",
             dict(
                 rsr_id=self.project.pk,
                 internal_id=self.activity.internal_id(),
                 iati_id=self.activity.iati_id(),
                 event=ERROR_PROJECT_DATA_INVALID,
                 extra=e,
             ))
Exemple #12
0
 def process(self):
     self._current_image()
     self._current_image_caption()
     self._current_image_credit()
     self._category()
     self._reporting_org()
     self._publish()
     try:
         self.project.full_clean()
     except ValidationError, e:
         log(
            "Warning: data validation error when saving project ID: {rsr_id}:\n{extra}",
            dict(
                rsr_id=self.project.pk,
                internal_id=self.activity.internal_id(),
                iati_id=self.activity.iati_id(),
                event=ERROR_PROJECT_DATA_INVALID,
                extra=e,
            )
         )
Exemple #13
0
 def _publish(self):
     has_image = self.project.current_image != ''
     has_lat_long = self.project.primary_location.latitude != 0 or self.project.primary_location.longitude != 0
     if has_image and has_lat_long:
         self.project.publishingstatus.status = PublishingStatus.STATUS_PUBLISHED
         log(
             "Project ID: {rsr_id} published",
             dict(
                 rsr_id=self.project.pk,
                 internal_id=self.activity.internal_id(),
                 iati_id=self.activity.iati_id(),
                 event=ACTION_PROJECT_PUBLISHED,
             ))
     else:
         self.project.publishingstatus.status = PublishingStatus.STATUS_UNPUBLISHED
         log(
             "Project ID: {rsr_id} not published",
             dict(
                 rsr_id=self.project.pk,
                 internal_id=self.activity.internal_id(),
                 iati_id=self.activity.iati_id(),
                 event=ACTION_PROJECT_NOT_PUBLISHED,
             ))
     self.project.publishingstatus.save()
Exemple #14
0
                        extra=msg.format(internal_id=activity.internal_id())
                    )
                )
        elif activity.iati_id():
            try:
                self.project = Project.objects.get(
                    iati_activity_id=activity.iati_id(), partnerships__organisation=RAIN_ORG_ID
                )
                return
            except:
                msg = "Could not find project or multiple projects found with IATI ID: {iati_id}"
                log(
                    msg,
                    dict(
                        rsr_id=self.activity.rsr_id(),
                        internal_id=self.activity.internal_id(),
                        iati_id=self.activity.iati_id(),
                        event=ERROR_IMAGE_UPLOAD,
                        extra=msg.format(iati_id=activity.iati_id())
                    )
                )
        raise Project.DoesNotExist


    def _current_image(self):
        image = ImageImporter(self.activity.current_image())
        try:
            image.get_image()
        except Exception, e:
            log(
                "Error trying to fetch image to project. Image URL: {extra}",
                dict(
def upload_organisations(argv):
    user_cred = credentials_from_args(argv)
    if user_cred:
        xml = load_xml(RAIN_ORGANISATIONS_URL)
        if xml:
            save_xml(xml, "rain_organisations_{datetime}.xml")
            parser = etree.XMLParser(ns_clean=True, recover=True, encoding='utf-8')
            root = etree.fromstring(xml, parser=parser)
            organisations = root.findall('organisation')
            for i, org_as_etree in enumerate(organisations):
                org_id = find_text(org_as_etree, 'internal_org_id@{{{}}}akvo_identifier'.format(RAIN_ORGANISATION_NS))
                iati_org_id = find_text(org_as_etree, 'iati_org_id')
                internal_org_id = find_text(org_as_etree, 'internal_org_id')
                name = find_text(org_as_etree, 'name')
                assert org_id or ( iati_org_id or internal_org_id), \
                    "Cannot create a new organisation without either an IATI ID or an internal ID"

                print "Processing organisation {}".format(name),
                try:
                    pk, content_owner_id = find_org(user_cred, RAIN_ORG_ID, org_id, iati_org_id, internal_org_id, name)
                except Exception, e:
                    message = "Error trying to find organisation {name} ID {pk}, error message: {extra}"
                    data = dict(
                        pk=org_id,
                        name=name,
                        iati_org_id=iati_org_id,
                        internal_org_id=internal_org_id,
                        event=ERROR_UPLOAD_ORG,
                        extra=e.message,
                    )
                    log(message, data)
                    print message.format(**data)
                    continue
                if content_owner_id is not None and int(content_owner_id) != RAIN_ORG_ID:
                    # Don't mess with orgs managed by others
                    message = "Organisation {name} ID {pk} is managed by another organisation: {extra}"
                    data = dict(
                        pk=org_id,
                        name=name,
                        iati_org_id=iati_org_id,
                        internal_org_id=internal_org_id,
                        event=ERROR_OTHER_CONTENT_OWNER,
                        extra=content_owner_id,
                    )
                    log(message, data)
                    print message.format(**data)
                    continue
                try:
                    org_as_dict = python_organisation(org_as_etree)
                except Exception, e:
                    message = "Error converting organisation XML to json. Error message: {extra}"
                    data = dict(
                        pk=org_id,
                        name=name,
                        iati_org_id=iati_org_id,
                        internal_org_id=internal_org_id,
                        event=ERROR_XML_PARSING,
                        extra=e.message,
                    )
                if pk:
                    ok, message, data = put_org(pk, internal_org_id, org_as_dict, user_cred)
                    log(message, data)
                    print message.format(**data)
                else:
                    ok, message, data = post_org(internal_org_id, org_as_dict, user_cred)
                    log(message, data)
                    print message.format(**data)
                if ok:
                    message, data = post_internal_id(user_cred, RAIN_ORG_ID, internal_org_id, data['pk'])
                    log(message, data)
                    print message.format(**data)
def upload_activities(argv):
    user = credentials_from_args(argv)
    if user:
        xml = load_xml(RAIN_IATI_ACTIVITES_URL)
        if xml:
            save_xml(xml, "rain_activities_{datetime}.xml")
            parser = etree.XMLParser(ns_clean=True,
                                     recover=True,
                                     encoding='utf-8')
            root = etree.fromstring(xml, parser=parser)
            document_akvo_ns = root.nsmap['akvo']
            assert document_akvo_ns == AKVO_NS, "Akvo name space is incorrect in the IATI XML"
            activities = root.findall('iati-activity')
            activity_count = len(activities)
            for i, activity in enumerate(activities):
                activity = RainActivity(activity, RAIN_ACTIVITY_NS, AKVO_NS)
                internal_id = activity.internal_id()
                iati_id = activity.iati_id()
                try:
                    assert iati_id is not None, "No IATI ID found, for activity number {} in the XML".format(
                        i + 1)
                except AssertionError, e:
                    message = "No IATI ID for activity number {extra}"
                    data = dict(
                        event=ERROR_MISSING_IATI_ID,
                        extra=i,
                    )
                    log(message, data)
                    print message.format(**data)
                    continue
                rsr_id = activity.rsr_id()
                print "({current} of {activity_count}) Processing activity {iati_id}".format(
                    current=i + 1,
                    activity_count=activity_count,
                    iati_id=iati_id),
                if len(activity.tree.findall('participating-org')) > 0:
                    try:
                        rsr_id = identify_rsr_project(user, rsr_id, iati_id,
                                                      internal_id)
                    except AssertionError, e:
                        message = "Error identifying RSR project: ID {pk}, IATI ID: {iati_id}, Error message: \n{extra}"
                        data = dict(
                            pk=rsr_id,
                            iati_id=iati_id,
                            event=ERROR_IDENTIFY_RSR_PROJECT,
                            extra=e.message,
                        )
                        log(message, data)
                        print message.format(**data)
                        continue
                    # HACK! removing duplicates of RAIN as participating-org
                    for dupe_rain in activity.tree.xpath(
                            'participating-org[@akvo:ref="0"]',
                            namespaces={
                                'akvo': 'http://akvo.org/iati-activities'
                            }):
                        dupe_rain.getparent().remove(dupe_rain)
                    if rsr_id:
                        ok, message, data = put_an_activity(
                            activity.tree, rsr_id, user)
                        log(message, data)
                        print message.format(**data)
                    else:
                        ok, message, data = post_an_activity(
                            activity.tree, user)
                        log(message, data)
                        print message.format(**data)
                else:
                    message = "Iati-activity {iati_id} has no participating-orgs, aborting"
                    data = dict(
                        iati_id=iati_id,
                        event=ERROR_NO_ORGS,
                    )
                    log(message, data)
                    print(message.format(**data))
Exemple #17
0
def upload_organisations(argv):
    user_cred = credentials_from_args(argv)
    if user_cred:
        xml = load_xml(RAIN_ORGANISATIONS_URL)
        if xml:
            save_xml(xml, "rain_organisations_{datetime}.xml")
            parser = etree.XMLParser(ns_clean=True,
                                     recover=True,
                                     encoding='utf-8')
            root = etree.fromstring(xml, parser=parser)
            organisations = root.findall('organisation')
            for i, org_as_etree in enumerate(organisations):
                org_id = find_text(
                    org_as_etree,
                    'internal_org_id@{{{}}}akvo_identifier'.format(
                        RAIN_ORGANISATION_NS))
                iati_org_id = find_text(org_as_etree, 'iati_org_id')
                internal_org_id = find_text(org_as_etree, 'internal_org_id')
                name = find_text(org_as_etree, 'name')
                assert org_id or ( iati_org_id or internal_org_id), \
                    "Cannot create a new organisation without either an IATI ID or an internal ID"

                print "Processing organisation {}".format(name),
                try:
                    pk, content_owner_id = find_org(user_cred, RAIN_ORG_ID,
                                                    org_id, iati_org_id,
                                                    internal_org_id, name)
                except Exception, e:
                    message = "Error trying to find organisation {name} ID {pk}, error message: {extra}"
                    data = dict(
                        pk=org_id,
                        name=name,
                        iati_org_id=iati_org_id,
                        internal_org_id=internal_org_id,
                        event=ERROR_UPLOAD_ORG,
                        extra=e.message,
                    )
                    log(message, data)
                    print message.format(**data)
                    continue
                if content_owner_id is not None and int(
                        content_owner_id) != RAIN_ORG_ID:
                    # Don't mess with orgs managed by others
                    message = "Organisation {name} ID {pk} is managed by another organisation: {extra}"
                    data = dict(
                        pk=org_id,
                        name=name,
                        iati_org_id=iati_org_id,
                        internal_org_id=internal_org_id,
                        event=ERROR_OTHER_CONTENT_OWNER,
                        extra=content_owner_id,
                    )
                    log(message, data)
                    print message.format(**data)
                    continue
                try:
                    org_as_dict = python_organisation(org_as_etree)
                except Exception, e:
                    message = "Error converting organisation XML to json. Error message: {extra}"
                    data = dict(
                        pk=org_id,
                        name=name,
                        iati_org_id=iati_org_id,
                        internal_org_id=internal_org_id,
                        event=ERROR_XML_PARSING,
                        extra=e.message,
                    )
                if pk:
                    ok, message, data = put_org(pk, internal_org_id,
                                                org_as_dict, user_cred)
                    log(message, data)
                    print message.format(**data)
                else:
                    ok, message, data = post_org(internal_org_id, org_as_dict,
                                                 user_cred)
                    log(message, data)
                    print message.format(**data)
                if ok:
                    message, data = post_internal_id(user_cred, RAIN_ORG_ID,
                                                     internal_org_id,
                                                     data['pk'])
                    log(message, data)
                    print message.format(**data)
Exemple #18
0
                         internal_id=self.activity.internal_id(),
                         iati_id=self.activity.iati_id(),
                         event=ERROR_IMAGE_UPLOAD,
                         extra=msg.format(internal_id=activity.internal_id())))
        elif activity.iati_id():
            try:
                self.project = Project.objects.get(
                    iati_activity_id=activity.iati_id(),
                    partnerships__organisation=RAIN_ORG_ID)
                return
            except:
                msg = "Could not find project or multiple projects found with IATI ID: {iati_id}"
                log(
                    msg,
                    dict(rsr_id=self.activity.rsr_id(),
                         internal_id=self.activity.internal_id(),
                         iati_id=self.activity.iati_id(),
                         event=ERROR_IMAGE_UPLOAD,
                         extra=msg.format(iati_id=activity.iati_id())))
        raise Project.DoesNotExist

    def _current_image(self):
        image = ImageImporter(self.activity.current_image())
        try:
            image.get_image()
        except Exception, e:
            log(
                "Error trying to fetch image to project. Image URL: {extra}",
                dict(rsr_id=self.activity.rsr_id(),
                     internal_id=self.activity.internal_id(),
                     iati_id=self.activity.iati_id(),