def upload_activities(argv):
user = credentials_from_args(argv)
if user:
xml = load_xml(RAIN_IATI_ACTIVITES_URL)
if xml:
save_xml(xml, "rain_activities_{datetime}.xml")
parser = etree.XMLParser(ns_clean=True, recover=True, encoding="utf-8")
root = etree.fromstring(xml, parser=parser)
document_akvo_ns = root.nsmap["akvo"]
assert document_akvo_ns == AKVO_NS, "Akvo name space is incorrect in the IATI XML"
activities = root.findall("iati-activity")
activity_count = len(activities)
for i, activity in enumerate(activities):
activity = RainActivity(activity, RAIN_ACTIVITY_NS, AKVO_NS)
internal_id = activity.internal_id()
iati_id = activity.iati_id()
try:
assert iati_id is not None, "No IATI ID found, for activity number {} in the XML".format(i + 1)
except AssertionError, e:
message = "No IATI ID for activity number {extra}"
data = dict(event=ERROR_MISSING_IATI_ID, extra=i)
log(message, data)
print message.format(**data)
continue
rsr_id = activity.rsr_id()
print "({current} of {activity_count}) Processing activity {iati_id}".format(
current=i + 1, activity_count=activity_count, iati_id=iati_id
),
if len(activity.tree.findall("participating-org")) > 0:
try:
rsr_id = identify_rsr_project(user, rsr_id, iati_id, internal_id)
except AssertionError, e:
message = "Error identifying RSR project: ID {pk}, IATI ID: {iati_id}, Error message: \n{extra}"
data = dict(pk=rsr_id, iati_id=iati_id, event=ERROR_IDENTIFY_RSR_PROJECT, extra=e.message)
log(message, data)
print message.format(**data)
continue
# HACK! removing duplicates of RAIN as participating-org
for dupe_rain in activity.tree.xpath(
'participating-org[@akvo:ref="0"]', namespaces={"akvo": "http://akvo.org/iati-activities"}
):
dupe_rain.getparent().remove(dupe_rain)
if rsr_id:
ok, message, data = put_an_activity(activity.tree, rsr_id, user)
log(message, data)
print message.format(**data)
else:
ok, message, data = post_an_activity(activity.tree, user)
log(message, data)
print message.format(**data)
else:
message = "Iati-activity {iati_id} has no participating-orgs, aborting"
data = dict(iati_id=iati_id, event=ERROR_NO_ORGS)
log(message, data)
print (message.format(**data))
def setup(self):
""" Create a list where each list item is one Activity object
"""
self.activities = []
outsys("\nRunning {}() ".format(who_am_i()))
xml = load_xml(RAIN_IATI_ACTIVITES_URL)
if xml:
parser = etree.XMLParser(ns_clean=True, recover=True, encoding='utf-8')
iati_activities = etree.fromstring(xml, parser=parser)
iati_activity_list = iati_activities.xpath('//iati-activity')
for iati_activity in iati_activity_list:
outsys(".")
self.activities.append(RainActivity(iati_activity, RAIN_ACTIVITY_NS, AKVO_NS))
def setup(self):
""" Create a list where each list item is one Activity object
"""
self.activities = []
outsys("\nRunning {}() ".format(who_am_i()))
xml = load_xml(RAIN_IATI_ACTIVITES_URL)
if xml:
parser = etree.XMLParser(ns_clean=True,
recover=True,
encoding='utf-8')
iati_activities = etree.fromstring(xml, parser=parser)
iati_activity_list = iati_activities.xpath('//iati-activity')
for iati_activity in iati_activity_list:
outsys(".")
self.activities.append(
RainActivity(iati_activity, RAIN_ACTIVITY_NS, AKVO_NS))
def upload_organisations(argv):
user_cred = credentials_from_args(argv)
if user_cred:
xml = load_xml(RAIN_ORGANISATIONS_URL)
if xml:
save_xml(xml, "rain_organisations_{datetime}.xml")
parser = etree.XMLParser(ns_clean=True,
recover=True,
encoding='utf-8')
root = etree.fromstring(xml, parser=parser)
organisations = root.findall('organisation')
for i, org_as_etree in enumerate(organisations):
org_id = find_text(
org_as_etree,
'internal_org_id@{{{}}}akvo_identifier'.format(
RAIN_ORGANISATION_NS))
iati_org_id = find_text(org_as_etree, 'iati_org_id')
internal_org_id = find_text(org_as_etree, 'internal_org_id')
name = find_text(org_as_etree, 'name')
assert org_id or ( iati_org_id or internal_org_id), \
"Cannot create a new organisation without either an IATI ID or an internal ID"
print "Processing organisation {}".format(name),
try:
pk, content_owner_id = find_org(user_cred, RAIN_ORG_ID,
org_id, iati_org_id,
internal_org_id, name)
except Exception, e:
message = "Error trying to find organisation {name} ID {pk}, error message: {extra}"
data = dict(
pk=org_id,
name=name,
iati_org_id=iati_org_id,
internal_org_id=internal_org_id,
event=ERROR_UPLOAD_ORG,
extra=e.message,
)
log(message, data)
print message.format(**data)
continue
if content_owner_id is not None and int(
content_owner_id) != RAIN_ORG_ID:
# Don't mess with orgs managed by others
message = "Organisation {name} ID {pk} is managed by another organisation: {extra}"
data = dict(
pk=org_id,
name=name,
iati_org_id=iati_org_id,
internal_org_id=internal_org_id,
event=ERROR_OTHER_CONTENT_OWNER,
extra=content_owner_id,
)
log(message, data)
print message.format(**data)
continue
try:
org_as_dict = python_organisation(org_as_etree)
except Exception, e:
message = "Error converting organisation XML to json. Error message: {extra}"
data = dict(
pk=org_id,
name=name,
iati_org_id=iati_org_id,
internal_org_id=internal_org_id,
event=ERROR_XML_PARSING,
extra=e.message,
)
if pk:
ok, message, data = put_org(pk, internal_org_id,
org_as_dict, user_cred)
log(message, data)
print message.format(**data)
else:
ok, message, data = post_org(internal_org_id, org_as_dict,
user_cred)
log(message, data)
print message.format(**data)
if ok:
message, data = post_internal_id(user_cred, RAIN_ORG_ID,
internal_org_id,
data['pk'])
log(message, data)
print message.format(**data)
def upload_activities(argv):
user = credentials_from_args(argv)
if user:
xml = load_xml(RAIN_IATI_ACTIVITES_URL)
if xml:
save_xml(xml, "rain_activities_{datetime}.xml")
parser = etree.XMLParser(ns_clean=True,
recover=True,
encoding='utf-8')
root = etree.fromstring(xml, parser=parser)
document_akvo_ns = root.nsmap['akvo']
assert document_akvo_ns == AKVO_NS, "Akvo name space is incorrect in the IATI XML"
activities = root.findall('iati-activity')
activity_count = len(activities)
for i, activity in enumerate(activities):
activity = RainActivity(activity, RAIN_ACTIVITY_NS, AKVO_NS)
internal_id = activity.internal_id()
iati_id = activity.iati_id()
try:
assert iati_id is not None, "No IATI ID found, for activity number {} in the XML".format(
i + 1)
except AssertionError, e:
message = "No IATI ID for activity number {extra}"
data = dict(
event=ERROR_MISSING_IATI_ID,
extra=i,
)
log(message, data)
print message.format(**data)
continue
rsr_id = activity.rsr_id()
print "({current} of {activity_count}) Processing activity {iati_id}".format(
current=i + 1,
activity_count=activity_count,
iati_id=iati_id),
if len(activity.tree.findall('participating-org')) > 0:
try:
rsr_id = identify_rsr_project(user, rsr_id, iati_id,
internal_id)
except AssertionError, e:
message = "Error identifying RSR project: ID {pk}, IATI ID: {iati_id}, Error message: \n{extra}"
data = dict(
pk=rsr_id,
iati_id=iati_id,
event=ERROR_IDENTIFY_RSR_PROJECT,
extra=e.message,
)
log(message, data)
print message.format(**data)
continue
# HACK! removing duplicates of RAIN as participating-org
for dupe_rain in activity.tree.xpath(
'participating-org[@akvo:ref="0"]',
namespaces={
'akvo': 'http://akvo.org/iati-activities'
}):
dupe_rain.getparent().remove(dupe_rain)
if rsr_id:
ok, message, data = put_an_activity(
activity.tree, rsr_id, user)
log(message, data)
print message.format(**data)
else:
ok, message, data = post_an_activity(
activity.tree, user)
log(message, data)
print message.format(**data)
else:
message = "Iati-activity {iati_id} has no participating-orgs, aborting"
data = dict(
iati_id=iati_id,
event=ERROR_NO_ORGS,
)
log(message, data)
print(message.format(**data))
def upload_organisations(argv):
user_cred = credentials_from_args(argv)
if user_cred:
xml = load_xml(RAIN_ORGANISATIONS_URL)
if xml:
save_xml(xml, "rain_organisations_{datetime}.xml")
parser = etree.XMLParser(ns_clean=True, recover=True, encoding='utf-8')
root = etree.fromstring(xml, parser=parser)
organisations = root.findall('organisation')
for i, org_as_etree in enumerate(organisations):
org_id = find_text(org_as_etree, 'internal_org_id@{{{}}}akvo_identifier'.format(RAIN_ORGANISATION_NS))
iati_org_id = find_text(org_as_etree, 'iati_org_id')
internal_org_id = find_text(org_as_etree, 'internal_org_id')
name = find_text(org_as_etree, 'name')
assert org_id or ( iati_org_id or internal_org_id), \
"Cannot create a new organisation without either an IATI ID or an internal ID"
print "Processing organisation {}".format(name),
try:
pk, content_owner_id = find_org(user_cred, RAIN_ORG_ID, org_id, iati_org_id, internal_org_id, name)
except Exception, e:
message = "Error trying to find organisation {name} ID {pk}, error message: {extra}"
data = dict(
pk=org_id,
name=name,
iati_org_id=iati_org_id,
internal_org_id=internal_org_id,
event=ERROR_UPLOAD_ORG,
extra=e.message,
)
log(message, data)
print message.format(**data)
continue
if content_owner_id is not None and int(content_owner_id) != RAIN_ORG_ID:
# Don't mess with orgs managed by others
message = "Organisation {name} ID {pk} is managed by another organisation: {extra}"
data = dict(
pk=org_id,
name=name,
iati_org_id=iati_org_id,
internal_org_id=internal_org_id,
event=ERROR_OTHER_CONTENT_OWNER,
extra=content_owner_id,
)
log(message, data)
print message.format(**data)
continue
try:
org_as_dict = python_organisation(org_as_etree)
except Exception, e:
message = "Error converting organisation XML to json. Error message: {extra}"
data = dict(
pk=org_id,
name=name,
iati_org_id=iati_org_id,
internal_org_id=internal_org_id,
event=ERROR_XML_PARSING,
extra=e.message,
)
if pk:
ok, message, data = put_org(pk, internal_org_id, org_as_dict, user_cred)
log(message, data)
print message.format(**data)
else:
ok, message, data = post_org(internal_org_id, org_as_dict, user_cred)
log(message, data)
print message.format(**data)
if ok:
message, data = post_internal_id(user_cred, RAIN_ORG_ID, internal_org_id, data['pk'])
log(message, data)
print message.format(**data)
