def upload_activities(argv):
    user = credentials_from_args(argv)
    if user:
        xml = load_xml(RAIN_IATI_ACTIVITES_URL)
        if xml:
            save_xml(xml, "rain_activities_{datetime}.xml")
            parser = etree.XMLParser(ns_clean=True, recover=True, encoding="utf-8")
            root = etree.fromstring(xml, parser=parser)
            document_akvo_ns = root.nsmap["akvo"]
            assert document_akvo_ns == AKVO_NS, "Akvo name space is incorrect in the IATI XML"
            activities = root.findall("iati-activity")
            activity_count = len(activities)
            for i, activity in enumerate(activities):
                activity = RainActivity(activity, RAIN_ACTIVITY_NS, AKVO_NS)
                internal_id = activity.internal_id()
                iati_id = activity.iati_id()
                try:
                    assert iati_id is not None, "No IATI ID found, for activity number {} in the XML".format(i + 1)
                except AssertionError, e:
                    message = "No IATI ID for activity number {extra}"
                    data = dict(event=ERROR_MISSING_IATI_ID, extra=i)
                    log(message, data)
                    print message.format(**data)
                    continue
                rsr_id = activity.rsr_id()
                print "({current} of {activity_count}) Processing activity {iati_id}".format(
                    current=i + 1, activity_count=activity_count, iati_id=iati_id
                ),
                if len(activity.tree.findall("participating-org")) > 0:
                    try:
                        rsr_id = identify_rsr_project(user, rsr_id, iati_id, internal_id)
                    except AssertionError, e:
                        message = "Error identifying RSR project: ID {pk}, IATI ID: {iati_id}, Error message: \n{extra}"
                        data = dict(pk=rsr_id, iati_id=iati_id, event=ERROR_IDENTIFY_RSR_PROJECT, extra=e.message)
                        log(message, data)
                        print message.format(**data)
                        continue
                    # HACK! removing duplicates of RAIN as participating-org
                    for dupe_rain in activity.tree.xpath(
                        'participating-org[@akvo:ref="0"]', namespaces={"akvo": "http://akvo.org/iati-activities"}
                    ):
                        dupe_rain.getparent().remove(dupe_rain)
                    if rsr_id:
                        ok, message, data = put_an_activity(activity.tree, rsr_id, user)
                        log(message, data)
                        print message.format(**data)
                    else:
                        ok, message, data = post_an_activity(activity.tree, user)
                        log(message, data)
                        print message.format(**data)
                else:
                    message = "Iati-activity {iati_id} has no participating-orgs, aborting"
                    data = dict(iati_id=iati_id, event=ERROR_NO_ORGS)
                    log(message, data)
                    print (message.format(**data))
示例#2
0
def upload_organisations(argv):
    user_cred = credentials_from_args(argv)
    if user_cred:
        xml = load_xml(RAIN_ORGANISATIONS_URL)
        if xml:
            save_xml(xml, "rain_organisations_{datetime}.xml")
            parser = etree.XMLParser(ns_clean=True,
                                     recover=True,
                                     encoding='utf-8')
            root = etree.fromstring(xml, parser=parser)
            organisations = root.findall('organisation')
            for i, org_as_etree in enumerate(organisations):
                org_id = find_text(
                    org_as_etree,
                    'internal_org_id@{{{}}}akvo_identifier'.format(
                        RAIN_ORGANISATION_NS))
                iati_org_id = find_text(org_as_etree, 'iati_org_id')
                internal_org_id = find_text(org_as_etree, 'internal_org_id')
                name = find_text(org_as_etree, 'name')
                assert org_id or ( iati_org_id or internal_org_id), \
                    "Cannot create a new organisation without either an IATI ID or an internal ID"

                print "Processing organisation {}".format(name),
                try:
                    pk, content_owner_id = find_org(user_cred, RAIN_ORG_ID,
                                                    org_id, iati_org_id,
                                                    internal_org_id, name)
                except Exception, e:
                    message = "Error trying to find organisation {name} ID {pk}, error message: {extra}"
                    data = dict(
                        pk=org_id,
                        name=name,
                        iati_org_id=iati_org_id,
                        internal_org_id=internal_org_id,
                        event=ERROR_UPLOAD_ORG,
                        extra=e.message,
                    )
                    log(message, data)
                    print message.format(**data)
                    continue
                if content_owner_id is not None and int(
                        content_owner_id) != RAIN_ORG_ID:
                    # Don't mess with orgs managed by others
                    message = "Organisation {name} ID {pk} is managed by another organisation: {extra}"
                    data = dict(
                        pk=org_id,
                        name=name,
                        iati_org_id=iati_org_id,
                        internal_org_id=internal_org_id,
                        event=ERROR_OTHER_CONTENT_OWNER,
                        extra=content_owner_id,
                    )
                    log(message, data)
                    print message.format(**data)
                    continue
                try:
                    org_as_dict = python_organisation(org_as_etree)
                except Exception, e:
                    message = "Error converting organisation XML to json. Error message: {extra}"
                    data = dict(
                        pk=org_id,
                        name=name,
                        iati_org_id=iati_org_id,
                        internal_org_id=internal_org_id,
                        event=ERROR_XML_PARSING,
                        extra=e.message,
                    )
                if pk:
                    ok, message, data = put_org(pk, internal_org_id,
                                                org_as_dict, user_cred)
                    log(message, data)
                    print message.format(**data)
                else:
                    ok, message, data = post_org(internal_org_id, org_as_dict,
                                                 user_cred)
                    log(message, data)
                    print message.format(**data)
                if ok:
                    message, data = post_internal_id(user_cred, RAIN_ORG_ID,
                                                     internal_org_id,
                                                     data['pk'])
                    log(message, data)
                    print message.format(**data)
def upload_activities(argv):
    user = credentials_from_args(argv)
    if user:
        xml = load_xml(RAIN_IATI_ACTIVITES_URL)
        if xml:
            save_xml(xml, "rain_activities_{datetime}.xml")
            parser = etree.XMLParser(ns_clean=True,
                                     recover=True,
                                     encoding='utf-8')
            root = etree.fromstring(xml, parser=parser)
            document_akvo_ns = root.nsmap['akvo']
            assert document_akvo_ns == AKVO_NS, "Akvo name space is incorrect in the IATI XML"
            activities = root.findall('iati-activity')
            activity_count = len(activities)
            for i, activity in enumerate(activities):
                activity = RainActivity(activity, RAIN_ACTIVITY_NS, AKVO_NS)
                internal_id = activity.internal_id()
                iati_id = activity.iati_id()
                try:
                    assert iati_id is not None, "No IATI ID found, for activity number {} in the XML".format(
                        i + 1)
                except AssertionError, e:
                    message = "No IATI ID for activity number {extra}"
                    data = dict(
                        event=ERROR_MISSING_IATI_ID,
                        extra=i,
                    )
                    log(message, data)
                    print message.format(**data)
                    continue
                rsr_id = activity.rsr_id()
                print "({current} of {activity_count}) Processing activity {iati_id}".format(
                    current=i + 1,
                    activity_count=activity_count,
                    iati_id=iati_id),
                if len(activity.tree.findall('participating-org')) > 0:
                    try:
                        rsr_id = identify_rsr_project(user, rsr_id, iati_id,
                                                      internal_id)
                    except AssertionError, e:
                        message = "Error identifying RSR project: ID {pk}, IATI ID: {iati_id}, Error message: \n{extra}"
                        data = dict(
                            pk=rsr_id,
                            iati_id=iati_id,
                            event=ERROR_IDENTIFY_RSR_PROJECT,
                            extra=e.message,
                        )
                        log(message, data)
                        print message.format(**data)
                        continue
                    # HACK! removing duplicates of RAIN as participating-org
                    for dupe_rain in activity.tree.xpath(
                            'participating-org[@akvo:ref="0"]',
                            namespaces={
                                'akvo': 'http://akvo.org/iati-activities'
                            }):
                        dupe_rain.getparent().remove(dupe_rain)
                    if rsr_id:
                        ok, message, data = put_an_activity(
                            activity.tree, rsr_id, user)
                        log(message, data)
                        print message.format(**data)
                    else:
                        ok, message, data = post_an_activity(
                            activity.tree, user)
                        log(message, data)
                        print message.format(**data)
                else:
                    message = "Iati-activity {iati_id} has no participating-orgs, aborting"
                    data = dict(
                        iati_id=iati_id,
                        event=ERROR_NO_ORGS,
                    )
                    log(message, data)
                    print(message.format(**data))
示例#4
0
def upload_organisations(argv):
    user_cred = credentials_from_args(argv)
    if user_cred:
        xml = load_xml(RAIN_ORGANISATIONS_URL)
        if xml:
            save_xml(xml, "rain_organisations_{datetime}.xml")
            parser = etree.XMLParser(ns_clean=True, recover=True, encoding='utf-8')
            root = etree.fromstring(xml, parser=parser)
            organisations = root.findall('organisation')
            for i, org_as_etree in enumerate(organisations):
                org_id = find_text(org_as_etree, 'internal_org_id@{{{}}}akvo_identifier'.format(RAIN_ORGANISATION_NS))
                iati_org_id = find_text(org_as_etree, 'iati_org_id')
                internal_org_id = find_text(org_as_etree, 'internal_org_id')
                name = find_text(org_as_etree, 'name')
                assert org_id or ( iati_org_id or internal_org_id), \
                    "Cannot create a new organisation without either an IATI ID or an internal ID"

                print "Processing organisation {}".format(name),
                try:
                    pk, content_owner_id = find_org(user_cred, RAIN_ORG_ID, org_id, iati_org_id, internal_org_id, name)
                except Exception, e:
                    message = "Error trying to find organisation {name} ID {pk}, error message: {extra}"
                    data = dict(
                        pk=org_id,
                        name=name,
                        iati_org_id=iati_org_id,
                        internal_org_id=internal_org_id,
                        event=ERROR_UPLOAD_ORG,
                        extra=e.message,
                    )
                    log(message, data)
                    print message.format(**data)
                    continue
                if content_owner_id is not None and int(content_owner_id) != RAIN_ORG_ID:
                    # Don't mess with orgs managed by others
                    message = "Organisation {name} ID {pk} is managed by another organisation: {extra}"
                    data = dict(
                        pk=org_id,
                        name=name,
                        iati_org_id=iati_org_id,
                        internal_org_id=internal_org_id,
                        event=ERROR_OTHER_CONTENT_OWNER,
                        extra=content_owner_id,
                    )
                    log(message, data)
                    print message.format(**data)
                    continue
                try:
                    org_as_dict = python_organisation(org_as_etree)
                except Exception, e:
                    message = "Error converting organisation XML to json. Error message: {extra}"
                    data = dict(
                        pk=org_id,
                        name=name,
                        iati_org_id=iati_org_id,
                        internal_org_id=internal_org_id,
                        event=ERROR_XML_PARSING,
                        extra=e.message,
                    )
                if pk:
                    ok, message, data = put_org(pk, internal_org_id, org_as_dict, user_cred)
                    log(message, data)
                    print message.format(**data)
                else:
                    ok, message, data = post_org(internal_org_id, org_as_dict, user_cred)
                    log(message, data)
                    print message.format(**data)
                if ok:
                    message, data = post_internal_id(user_cred, RAIN_ORG_ID, internal_org_id, data['pk'])
                    log(message, data)
                    print message.format(**data)