class ProjectSaver():
""" Perform the post import saving steps for the project, all related to the project image
"""
def __init__(self, activity, importing_org):
self.activity = activity
if activity.iati_id():
try:
self.project = Project.objects.get(
iati_activity_id=activity.iati_id()
)
return
except:
msg = "Could not find project or multiple projects found with IATI ID: {iati_id}"
log(
msg,
dict(
iati_id=self.activity.iati_id(),
)
)
raise Project.DoesNotExist
def _current_image(self):
image = ImageImporter(RVO_DEFAULT_IMAGE)
try:
image.get_image()
except Exception, e:
log(
"Error trying to fetch image to project. Image URL: {extra}",
dict(
iati_id=self.activity.iati_id(),
event=ERROR_IMAGE_UPLOAD,
extra=self.activity.current_image()
)
)
if image.image:
filename = model_and_instance_based_filename(
'Project', self.project.pk, 'current_image', image.filename
)
image_temp = NamedTemporaryFile(delete=True)
image_temp.write(image.image)
image_temp.flush()
self.project.current_image.save(filename, File(image_temp), save=True)
log(
"Save project image: {extra}",
dict(
iati_id=self.activity.iati_id(),
event=ACTION_SET_IMAGE,
extra=filename
)
)
else:
log(
"No image found for project: {rsr_id}",
dict(
iati_id=self.activity.iati_id(),
event=ERROR_IMAGE_NOT_FOUND,
)
)
def _publish(self):
self.project.publishingstatus.status = PublishingStatus.STATUS_PUBLISHED
log(
"Project ID: {iati_id} published",
dict(
iati_id=self.activity.iati_id(),
event=ACTION_PROJECT_PUBLISHED,
)
)
self.project.publishingstatus.save()
def _publish(self):
self.project.publishingstatus.status = PublishingStatus.STATUS_PUBLISHED
log(
"Project ID: {iati_id} published",
dict(
iati_id=self.activity.iati_id(),
event=ACTION_PROJECT_PUBLISHED,
)
)
self.project.publishingstatus.save()
def _process_project(self, activity):
try:
project_saver = ProjectSaver(activity, self.importing_org)
except Project.DoesNotExist, e:
log(
"Couldn't find project IATI ID: {iati_id}",
dict(
iati_id=activity.iati_id(),
event=ERROR_PROJECT_NOT_FOUND,
)
)
return None
def _process_project(self, activity):
try:
project_saver = ProjectSaver(activity, self.importing_org)
except Project.DoesNotExist, e:
log(
"Couldn't find project IATI ID: {iati_id}",
dict(
iati_id=activity.iati_id(),
event=ERROR_PROJECT_NOT_FOUND,
)
)
return None
def _current_image(self):
image = ImageImporter(RVO_DEFAULT_IMAGE)
try:
image.get_image()
except Exception, e:
log(
"Error trying to fetch image to project. Image URL: {extra}",
dict(
iati_id=self.activity.iati_id(),
event=ERROR_IMAGE_UPLOAD,
extra=self.activity.current_image()
)
)
def _current_image(self):
image = ImageImporter(RVO_DEFAULT_IMAGE)
try:
image.get_image()
except Exception, e:
log(
"Error trying to fetch image to project. Image URL: {extra}",
dict(
iati_id=self.activity.iati_id(),
event=ERROR_IMAGE_UPLOAD,
extra=self.activity.current_image()
)
)
def process(self):
self._sync_owner()
self._current_image()
self._publish()
self._keywords()
try:
self.project.full_clean()
except ValidationError, e:
log(
"Warning: data validation error when saving project ID: {iati_id}:\n{extra}",
dict(
iati_id=self.activity.iati_id(),
event=ERROR_PROJECT_DATA_INVALID,
extra=e,
)
)
def process(self):
self._reporting_org()
self._current_image()
self._publish()
self._keywords()
try:
self.project.full_clean()
except ValidationError, e:
log(
"Warning: data validation error when saving project ID: {iati_id}:\n{extra}",
dict(
iati_id=self.activity.iati_id(),
event=ERROR_PROJECT_DATA_INVALID,
extra=e,
)
)
def __init__(self, activity, importing_org):
self.activity = activity
if activity.iati_id():
try:
self.project = Project.objects.get(
iati_activity_id=activity.iati_id()
)
return
except:
msg = "Could not find project or multiple projects found with IATI ID: {iati_id}"
log(
msg,
dict(
iati_id=self.activity.iati_id(),
)
)
raise Project.DoesNotExist
def __init__(self, activity, importing_org):
self.activity = activity
if activity.iati_id():
try:
self.project = Project.objects.get(
iati_activity_id=activity.iati_id()
)
return
except:
msg = "Could not find project or multiple projects found with IATI ID: {iati_id}"
log(
msg,
dict(
iati_id=self.activity.iati_id(),
)
)
raise Project.DoesNotExist
def upload_activities(argv):
user = credentials_from_args(argv)
if user:
xml = load_xml(RVO_IATI_ACTIVITES_URL)
if xml:
save_xml(xml, "rain_activities_{datetime}.xml")
parser = etree.XMLParser(ns_clean=True,
recover=True,
encoding='utf-8')
root = etree.fromstring(xml, parser=parser)
activities = root.findall('iati-activity')
activity_count = len(activities)
for i, activity in enumerate(activities):
activity = RvoActivity(activity)
iati_id = activity.iati_id()
try:
assert iati_id is not None, "No IATI ID found, for activity number {} in the XML".format(
i + 1)
except AssertionError, e:
message = "No IATI ID for activity number {extra}"
data = dict(
event=ERROR_MISSING_IATI_ID,
extra=i,
)
log(message, data)
print message.format(**data)
continue
print "({current} of {activity_count}) Processing activity {iati_id}".format(
current=i + 1,
activity_count=activity_count,
iati_id=iati_id),
try:
rsr_id = identify_rsr_project(user, iati_id)
except AssertionError, e:
message = "Error identifying RSR project: IATI ID: {iati_id}, Error message: \n{extra}"
data = dict(
iati_id=iati_id,
event=ERROR_IDENTIFY_RSR_PROJECT,
extra=e.message,
)
log(message, data)
print message.format(**data)
continue
# HACK! trim titles
for title in activity.tree.xpath('title'):
if title.text and len(title.text) > 45:
title.text = title.text[:45]
# HACK! add Akvo NS to description and trim descriptions
for description in activity.tree.xpath(
'description[@type="1"]'):
description.attrib['{' + AKVO_NS + '}type'] = '5'
if description.text and len(description.text) > 400:
new_description = etree.SubElement(
activity.tree, "description")
new_description.text = description.text
new_description.attrib['type'] = '1'
new_description.attrib['{' + AKVO_NS + '}type'] = '6'
description.text = description.text[:400]
for goals_overview in activity.tree.xpath(
'description[@type="2"]'):
goals_overview.attrib['{' + AKVO_NS + '}type'] = '8'
if goals_overview.text and len(goals_overview.text) > 600:
goals_overview.text = goals_overview.text[:600]
# HACK! add locations
for location in activity.tree.xpath('location'):
if activity.tree.xpath('recipient-country'):
country = activity.tree.xpath(
'recipient-country')[0].get("code")
administrative = etree.SubElement(
location, "administrative")
administrative.attrib['country'] = country.lower()
# HACK! add budgets
for budget in activity.tree.xpath('budget'):
if budget.xpath('value'):
for value in budget.xpath('value'):
if value.text and len(value.text) > 8:
budget.getparent().remove(budget)
else:
value.attrib['{' + AKVO_NS + '}type'] = '14'
else:
budget.getparent().remove(budget)
# HACK! removing participating-orgs
for org in activity.tree.xpath('participating-org'):
org.getparent().remove(org)
# HACK! removing results
for result in activity.tree.xpath('result'):
result.getparent().remove(result)
if rsr_id:
ok, message, data = put_an_activity(
activity.tree, rsr_id, user)
log(message, data)
print message.format(**data)
else:
ok, message, data = post_an_activity(activity.tree, user)
log(message, data)
print message.format(**data)
except ValidationError, e:
log(
"Warning: data validation error when saving project ID: {iati_id}:\n{extra}",
dict(
iati_id=self.activity.iati_id(),
event=ERROR_PROJECT_DATA_INVALID,
extra=e,
)
)
# return
try:
self.project.save()
log(
"Saved project IATI ID: {iati_id}:\n{extra}",
dict(
iati_id=self.activity.iati_id(),
event=ACTION_PROJECT_POST_PROCESS_DONE,
)
)
except Exception, e:
log(
"Couldn't save project ID: {iati_id}:\n{extra}",
dict(
iati_id=self.activity.iati_id(),
event=ERROR_PROJECT_NOT_SAVED,
extra=e,
)
)
return
def upload_activities(argv):
user = credentials_from_args(argv)
if user:
xml = load_xml(RVO_IATI_ACTIVITES_URL)
if xml:
save_xml(xml, "rain_activities_{datetime}.xml")
parser = etree.XMLParser(ns_clean=True, recover=True, encoding='utf-8')
root = etree.fromstring(xml, parser=parser)
activities = root.findall('iati-activity')
activity_count = len(activities)
for i, activity in enumerate(activities):
activity = RvoActivity(activity)
iati_id = activity.iati_id()
try:
assert iati_id is not None, "No IATI ID found, for activity number {} in the XML".format(i+1)
except AssertionError, e:
message = "No IATI ID for activity number {extra}"
data = dict(
event=ERROR_MISSING_IATI_ID,
extra=i,
)
log(message, data)
print message.format(**data)
continue
print "({current} of {activity_count}) Processing activity {iati_id}".format(
current=i+1, activity_count=activity_count, iati_id=iati_id
),
try:
rsr_id = identify_rsr_project(user, iati_id)
except AssertionError, e:
message = "Error identifying RSR project: IATI ID: {iati_id}, Error message: \n{extra}"
data = dict(
iati_id=iati_id,
event=ERROR_IDENTIFY_RSR_PROJECT,
extra=e.message,
)
log(message, data)
print message.format(**data)
continue
# HACK! trim titles
for title in activity.tree.xpath('title'):
if title.text and len(title.text) > 45:
title.text = title.text[:45]
# HACK! add Akvo NS to description and trim descriptions
for description in activity.tree.xpath('description[@type="1"]'):
description.attrib['{' + AKVO_NS + '}type'] = '5'
if description.text and len(description.text) > 400:
new_description = etree.SubElement(activity.tree, "description")
new_description.text = description.text
new_description.attrib['type'] = '1'
new_description.attrib['{' + AKVO_NS + '}type'] = '6'
description.text = description.text[:400]
for goals_overview in activity.tree.xpath('description[@type="2"]'):
goals_overview.attrib['{' + AKVO_NS + '}type'] = '8'
if goals_overview.text and len(goals_overview.text) > 600:
goals_overview.text = goals_overview.text[:600]
# HACK! add locations
for location in activity.tree.xpath('location'):
if activity.tree.xpath('recipient-country'):
country = activity.tree.xpath('recipient-country')[0].get("code")
administrative = etree.SubElement(location, "administrative")
administrative.attrib['country'] = country.lower()
# HACK! add budgets
for budget in activity.tree.xpath('budget'):
if budget.xpath('value'):
for value in budget.xpath('value'):
if value.text and len(value.text) > 8:
budget.getparent().remove(budget)
else:
value.attrib['{' + AKVO_NS + '}type'] = '14'
else:
budget.getparent().remove(budget)
# HACK! removing participating-orgs
for org in activity.tree.xpath('participating-org'):
org.getparent().remove(org)
# HACK! removing results
for result in activity.tree.xpath('result'):
result.getparent().remove(result)
if rsr_id:
ok, message, data = put_an_activity(activity.tree, rsr_id, user)
log(message, data)
print message.format(**data)
else:
ok, message, data = post_an_activity(activity.tree, user)
log(message, data)
print message.format(**data)
except ValidationError, e:
log(
"Warning: data validation error when saving project ID: {iati_id}:\n{extra}",
dict(
iati_id=self.activity.iati_id(),
event=ERROR_PROJECT_DATA_INVALID,
extra=e,
)
)
# return
try:
self.project.save()
log(
"Saved project IATI ID: {iati_id}:\n{extra}",
dict(
iati_id=self.activity.iati_id(),
event=ACTION_PROJECT_POST_PROCESS_DONE,
)
)
except Exception, e:
log(
"Couldn't save project ID: {iati_id}:\n{extra}",
dict(
iati_id=self.activity.iati_id(),
event=ERROR_PROJECT_NOT_SAVED,
extra=e,
)
)
return