def setup(self):
""" Create a list where each list item is one Activity object
"""
self.activities = []
outsys("\nRunning {}() ".format(who_am_i()))
xml = load_xml(RVO_IATI_ACTIVITES_URL)
if xml:
parser = etree.XMLParser(ns_clean=True, recover=True, encoding='utf-8')
iati_activities = etree.fromstring(xml, parser=parser)
iati_activity_list = iati_activities.xpath('//iati-activity')
for iati_activity in iati_activity_list:
outsys(".")
self.activities.append(RvoActivity(iati_activity))
def setup(self):
""" Create a list where each list item is one Activity object
"""
self.activities = []
outsys("\nRunning {}() ".format(who_am_i()))
xml = load_xml(RVO_IATI_ACTIVITES_URL)
if xml:
parser = etree.XMLParser(ns_clean=True, recover=True, encoding='utf-8')
iati_activities = etree.fromstring(xml, parser=parser)
iati_activity_list = iati_activities.xpath('//iati-activity')
for iati_activity in iati_activity_list:
outsys(".")
self.activities.append(RvoActivity(iati_activity))
def upload_activities(argv):
user = credentials_from_args(argv)
if user:
xml = load_xml(RVO_IATI_ACTIVITES_URL)
if xml:
save_xml(xml, "rain_activities_{datetime}.xml")
parser = etree.XMLParser(ns_clean=True,
recover=True,
encoding='utf-8')
root = etree.fromstring(xml, parser=parser)
activities = root.findall('iati-activity')
activity_count = len(activities)
for i, activity in enumerate(activities):
activity = RvoActivity(activity)
iati_id = activity.iati_id()
try:
assert iati_id is not None, "No IATI ID found, for activity number {} in the XML".format(
i + 1)
except AssertionError, e:
message = "No IATI ID for activity number {extra}"
data = dict(
event=ERROR_MISSING_IATI_ID,
extra=i,
)
log(message, data)
print message.format(**data)
continue
print "({current} of {activity_count}) Processing activity {iati_id}".format(
current=i + 1,
activity_count=activity_count,
iati_id=iati_id),
try:
rsr_id = identify_rsr_project(user, iati_id)
except AssertionError, e:
message = "Error identifying RSR project: IATI ID: {iati_id}, Error message: \n{extra}"
data = dict(
iati_id=iati_id,
event=ERROR_IDENTIFY_RSR_PROJECT,
extra=e.message,
)
log(message, data)
print message.format(**data)
continue
# HACK! trim titles
for title in activity.tree.xpath('title'):
if title.text and len(title.text) > 45:
title.text = title.text[:45]
# HACK! add Akvo NS to description and trim descriptions
for description in activity.tree.xpath(
'description[@type="1"]'):
description.attrib['{' + AKVO_NS + '}type'] = '5'
if description.text and len(description.text) > 400:
new_description = etree.SubElement(
activity.tree, "description")
new_description.text = description.text
new_description.attrib['type'] = '1'
new_description.attrib['{' + AKVO_NS + '}type'] = '6'
description.text = description.text[:400]
for goals_overview in activity.tree.xpath(
'description[@type="2"]'):
goals_overview.attrib['{' + AKVO_NS + '}type'] = '8'
if goals_overview.text and len(goals_overview.text) > 600:
goals_overview.text = goals_overview.text[:600]
# HACK! add locations
for location in activity.tree.xpath('location'):
if activity.tree.xpath('recipient-country'):
country = activity.tree.xpath(
'recipient-country')[0].get("code")
administrative = etree.SubElement(
location, "administrative")
administrative.attrib['country'] = country.lower()
# HACK! add budgets
for budget in activity.tree.xpath('budget'):
if budget.xpath('value'):
for value in budget.xpath('value'):
if value.text and len(value.text) > 8:
budget.getparent().remove(budget)
else:
value.attrib['{' + AKVO_NS + '}type'] = '14'
else:
budget.getparent().remove(budget)
# HACK! removing participating-orgs
for org in activity.tree.xpath('participating-org'):
org.getparent().remove(org)
# HACK! removing results
for result in activity.tree.xpath('result'):
result.getparent().remove(result)
if rsr_id:
ok, message, data = put_an_activity(
activity.tree, rsr_id, user)
log(message, data)
print message.format(**data)
else:
ok, message, data = post_an_activity(activity.tree, user)
log(message, data)
print message.format(**data)
def upload_activities(argv):
user = credentials_from_args(argv)
if user:
xml = load_xml(RVO_IATI_ACTIVITES_URL)
if xml:
save_xml(xml, "rain_activities_{datetime}.xml")
parser = etree.XMLParser(ns_clean=True, recover=True, encoding='utf-8')
root = etree.fromstring(xml, parser=parser)
activities = root.findall('iati-activity')
activity_count = len(activities)
for i, activity in enumerate(activities):
activity = RvoActivity(activity)
iati_id = activity.iati_id()
try:
assert iati_id is not None, "No IATI ID found, for activity number {} in the XML".format(i+1)
except AssertionError, e:
message = "No IATI ID for activity number {extra}"
data = dict(
event=ERROR_MISSING_IATI_ID,
extra=i,
)
log(message, data)
print message.format(**data)
continue
print "({current} of {activity_count}) Processing activity {iati_id}".format(
current=i+1, activity_count=activity_count, iati_id=iati_id
),
try:
rsr_id = identify_rsr_project(user, iati_id)
except AssertionError, e:
message = "Error identifying RSR project: IATI ID: {iati_id}, Error message: \n{extra}"
data = dict(
iati_id=iati_id,
event=ERROR_IDENTIFY_RSR_PROJECT,
extra=e.message,
)
log(message, data)
print message.format(**data)
continue
# HACK! trim titles
for title in activity.tree.xpath('title'):
if title.text and len(title.text) > 45:
title.text = title.text[:45]
# HACK! add Akvo NS to description and trim descriptions
for description in activity.tree.xpath('description[@type="1"]'):
description.attrib['{' + AKVO_NS + '}type'] = '5'
if description.text and len(description.text) > 400:
new_description = etree.SubElement(activity.tree, "description")
new_description.text = description.text
new_description.attrib['type'] = '1'
new_description.attrib['{' + AKVO_NS + '}type'] = '6'
description.text = description.text[:400]
for goals_overview in activity.tree.xpath('description[@type="2"]'):
goals_overview.attrib['{' + AKVO_NS + '}type'] = '8'
if goals_overview.text and len(goals_overview.text) > 600:
goals_overview.text = goals_overview.text[:600]
# HACK! add locations
for location in activity.tree.xpath('location'):
if activity.tree.xpath('recipient-country'):
country = activity.tree.xpath('recipient-country')[0].get("code")
administrative = etree.SubElement(location, "administrative")
administrative.attrib['country'] = country.lower()
# HACK! add budgets
for budget in activity.tree.xpath('budget'):
if budget.xpath('value'):
for value in budget.xpath('value'):
if value.text and len(value.text) > 8:
budget.getparent().remove(budget)
else:
value.attrib['{' + AKVO_NS + '}type'] = '14'
else:
budget.getparent().remove(budget)
# HACK! removing participating-orgs
for org in activity.tree.xpath('participating-org'):
org.getparent().remove(org)
# HACK! removing results
for result in activity.tree.xpath('result'):
result.getparent().remove(result)
if rsr_id:
ok, message, data = put_an_activity(activity.tree, rsr_id, user)
log(message, data)
print message.format(**data)
else:
ok, message, data = post_an_activity(activity.tree, user)
log(message, data)
print message.format(**data)