def _publish(self): has_image = self.project.current_image != '' has_lat_long = self.project.primary_location.latitude != 0 or self.project.primary_location.longitude != 0 if has_image and has_lat_long: self.project.publishingstatus.status = PublishingStatus.STATUS_PUBLISHED log( "Project ID: {rsr_id} published", dict( rsr_id=self.project.pk, internal_id=self.activity.internal_id(), iati_id=self.activity.iati_id(), event=ACTION_PROJECT_PUBLISHED, ) ) else: self.project.publishingstatus.status = PublishingStatus.STATUS_UNPUBLISHED log( "Project ID: {rsr_id} not published", dict( rsr_id=self.project.pk, internal_id=self.activity.internal_id(), iati_id=self.activity.iati_id(), event=ACTION_PROJECT_NOT_PUBLISHED, ) ) self.project.publishingstatus.save()
def upload_activities(argv): user = credentials_from_args(argv) if user: xml = load_xml(RAIN_IATI_ACTIVITES_URL) if xml: save_xml(xml, "rain_activities_{datetime}.xml") parser = etree.XMLParser(ns_clean=True, recover=True, encoding="utf-8") root = etree.fromstring(xml, parser=parser) document_akvo_ns = root.nsmap["akvo"] assert document_akvo_ns == AKVO_NS, "Akvo name space is incorrect in the IATI XML" activities = root.findall("iati-activity") activity_count = len(activities) for i, activity in enumerate(activities): activity = RainActivity(activity, RAIN_ACTIVITY_NS, AKVO_NS) internal_id = activity.internal_id() iati_id = activity.iati_id() try: assert iati_id is not None, "No IATI ID found, for activity number {} in the XML".format(i + 1) except AssertionError, e: message = "No IATI ID for activity number {extra}" data = dict(event=ERROR_MISSING_IATI_ID, extra=i) log(message, data) print message.format(**data) continue rsr_id = activity.rsr_id() print "({current} of {activity_count}) Processing activity {iati_id}".format( current=i + 1, activity_count=activity_count, iati_id=iati_id ), if len(activity.tree.findall("participating-org")) > 0: try: rsr_id = identify_rsr_project(user, rsr_id, iati_id, internal_id) except AssertionError, e: message = "Error identifying RSR project: ID {pk}, IATI ID: {iati_id}, Error message: \n{extra}" data = dict(pk=rsr_id, iati_id=iati_id, event=ERROR_IDENTIFY_RSR_PROJECT, extra=e.message) log(message, data) print message.format(**data) continue # HACK! removing duplicates of RAIN as participating-org for dupe_rain in activity.tree.xpath( 'participating-org[@akvo:ref="0"]', namespaces={"akvo": "http://akvo.org/iati-activities"} ): dupe_rain.getparent().remove(dupe_rain) if rsr_id: ok, message, data = put_an_activity(activity.tree, rsr_id, user) log(message, data) print message.format(**data) else: ok, message, data = post_an_activity(activity.tree, user) log(message, data) print message.format(**data) else: message = "Iati-activity {iati_id} has no participating-orgs, aborting" data = dict(iati_id=iati_id, event=ERROR_NO_ORGS) log(message, data) print (message.format(**data))
def _current_image(self): image = ImageImporter(self.activity.current_image()) try: image.get_image() except Exception, e: log( "Error trying to fetch image to project. Image URL: {extra}", dict(rsr_id=self.activity.rsr_id(), internal_id=self.activity.internal_id(), iati_id=self.activity.iati_id(), event=ERROR_IMAGE_UPLOAD, extra=self.activity.current_image()))
def _process_project(self, activity): try: project_saver = ProjectSaver(activity, self.importing_org) except Project.DoesNotExist, e: log( "Couldn't find project ID: {rsr_id} IATI ID: {iati_id} internal ID: {internal_id}", dict( internal_id=activity.internal_id(), rsr_id=activity.rsr_id(), iati_id=activity.iati_id(), event=ERROR_PROJECT_NOT_FOUND, )) return None
def _process_project(self, activity): try: project_saver = ProjectSaver(activity, self.importing_org) except Project.DoesNotExist, e: log( "Couldn't find project ID: {rsr_id} IATI ID: {iati_id} internal ID: {internal_id}", dict( internal_id=activity.internal_id(), rsr_id=activity.rsr_id(), iati_id=activity.iati_id(), event=ERROR_PROJECT_NOT_FOUND, ) ) return None
def __init__(self, activity, importing_org): self.activity = activity if activity.rsr_id(): try: self.project = Project.objects.get(id=activity.rsr_id()) return except Project.DoesNotExist, e: msg = "Could not find project with ID: {rsr_id}" log( msg, dict(rsr_id=activity.rsr_id(), internal_id=activity.internal_id(), iati_id=activity.iati_id(), event=ERROR_IMAGE_UPLOAD, extra=msg.format(rsr_id=activity.rsr_id())))
def _current_image(self): image = ImageImporter(self.activity.current_image()) try: image.get_image() except Exception, e: log( "Error trying to fetch image to project. Image URL: {extra}", dict( rsr_id=self.activity.rsr_id(), internal_id=self.activity.internal_id(), iati_id=self.activity.iati_id(), event=ERROR_IMAGE_UPLOAD, extra=self.activity.current_image() ) )
class ProjectSaver(): """ Perform the post import saving steps for the project, all related to the project image """ def __init__(self, activity, importing_org): self.activity = activity if activity.rsr_id(): try: self.project = Project.objects.get(id=activity.rsr_id()) return except Project.DoesNotExist, e: msg = "Could not find project with ID: {rsr_id}" log( msg, dict(rsr_id=activity.rsr_id(), internal_id=activity.internal_id(), iati_id=activity.iati_id(), event=ERROR_IMAGE_UPLOAD, extra=msg.format(rsr_id=activity.rsr_id()))) elif activity.internal_id(): try: self.project = Project.objects.get( partnerships__internal_id=activity.internal_id(), partnerships__organisation=RAIN_ORG_ID) return except: msg = "Could not find project or multiple projects found with RAIN internal ID: {internal_id}" log( msg, dict(rsr_id=self.activity.rsr_id(), internal_id=self.activity.internal_id(), iati_id=self.activity.iati_id(), event=ERROR_IMAGE_UPLOAD, extra=msg.format(internal_id=activity.internal_id())))
def __init__(self, activity, importing_org): self.activity = activity if activity.rsr_id(): try: self.project = Project.objects.get(id=activity.rsr_id()) return except Project.DoesNotExist, e: msg = "Could not find project with ID: {rsr_id}" log( msg, dict( rsr_id=activity.rsr_id(), event=ERROR_IMAGE_UPLOAD, extra=msg.format(rsr_id=activity.rsr_id()) ) )
def process(self): self._current_image() self._current_image_caption() self._current_image_credit() try: self.project.full_clean() except ValidationError, e: log( "Data validation error when saving project ID: {rsr_id}:\n{extra}", dict( rsr_id=self.project.pk, event=ERROR_PROJECT_DATA_INVALID, extra=e, ) ) return
def process(self): self._current_image() self._current_image_caption() self._current_image_credit() self._category() self._sync_owner() self._publish() try: self.project.full_clean() except ValidationError, e: log( "Warning: data validation error when saving project ID: {rsr_id}:\n{extra}", dict( rsr_id=self.project.pk, internal_id=self.activity.internal_id(), iati_id=self.activity.iati_id(), event=ERROR_PROJECT_DATA_INVALID, extra=e, ))
def process(self): self._current_image() self._current_image_caption() self._current_image_credit() self._category() self._reporting_org() self._publish() try: self.project.full_clean() except ValidationError, e: log( "Warning: data validation error when saving project ID: {rsr_id}:\n{extra}", dict( rsr_id=self.project.pk, internal_id=self.activity.internal_id(), iati_id=self.activity.iati_id(), event=ERROR_PROJECT_DATA_INVALID, extra=e, ) )
def _publish(self): has_image = self.project.current_image != '' has_lat_long = self.project.primary_location.latitude != 0 or self.project.primary_location.longitude != 0 if has_image and has_lat_long: self.project.publishingstatus.status = PublishingStatus.STATUS_PUBLISHED log( "Project ID: {rsr_id} published", dict( rsr_id=self.project.pk, internal_id=self.activity.internal_id(), iati_id=self.activity.iati_id(), event=ACTION_PROJECT_PUBLISHED, )) else: self.project.publishingstatus.status = PublishingStatus.STATUS_UNPUBLISHED log( "Project ID: {rsr_id} not published", dict( rsr_id=self.project.pk, internal_id=self.activity.internal_id(), iati_id=self.activity.iati_id(), event=ACTION_PROJECT_NOT_PUBLISHED, )) self.project.publishingstatus.save()
extra=msg.format(internal_id=activity.internal_id()) ) ) elif activity.iati_id(): try: self.project = Project.objects.get( iati_activity_id=activity.iati_id(), partnerships__organisation=RAIN_ORG_ID ) return except: msg = "Could not find project or multiple projects found with IATI ID: {iati_id}" log( msg, dict( rsr_id=self.activity.rsr_id(), internal_id=self.activity.internal_id(), iati_id=self.activity.iati_id(), event=ERROR_IMAGE_UPLOAD, extra=msg.format(iati_id=activity.iati_id()) ) ) raise Project.DoesNotExist def _current_image(self): image = ImageImporter(self.activity.current_image()) try: image.get_image() except Exception, e: log( "Error trying to fetch image to project. Image URL: {extra}", dict(
def upload_organisations(argv): user_cred = credentials_from_args(argv) if user_cred: xml = load_xml(RAIN_ORGANISATIONS_URL) if xml: save_xml(xml, "rain_organisations_{datetime}.xml") parser = etree.XMLParser(ns_clean=True, recover=True, encoding='utf-8') root = etree.fromstring(xml, parser=parser) organisations = root.findall('organisation') for i, org_as_etree in enumerate(organisations): org_id = find_text(org_as_etree, 'internal_org_id@{{{}}}akvo_identifier'.format(RAIN_ORGANISATION_NS)) iati_org_id = find_text(org_as_etree, 'iati_org_id') internal_org_id = find_text(org_as_etree, 'internal_org_id') name = find_text(org_as_etree, 'name') assert org_id or ( iati_org_id or internal_org_id), \ "Cannot create a new organisation without either an IATI ID or an internal ID" print "Processing organisation {}".format(name), try: pk, content_owner_id = find_org(user_cred, RAIN_ORG_ID, org_id, iati_org_id, internal_org_id, name) except Exception, e: message = "Error trying to find organisation {name} ID {pk}, error message: {extra}" data = dict( pk=org_id, name=name, iati_org_id=iati_org_id, internal_org_id=internal_org_id, event=ERROR_UPLOAD_ORG, extra=e.message, ) log(message, data) print message.format(**data) continue if content_owner_id is not None and int(content_owner_id) != RAIN_ORG_ID: # Don't mess with orgs managed by others message = "Organisation {name} ID {pk} is managed by another organisation: {extra}" data = dict( pk=org_id, name=name, iati_org_id=iati_org_id, internal_org_id=internal_org_id, event=ERROR_OTHER_CONTENT_OWNER, extra=content_owner_id, ) log(message, data) print message.format(**data) continue try: org_as_dict = python_organisation(org_as_etree) except Exception, e: message = "Error converting organisation XML to json. Error message: {extra}" data = dict( pk=org_id, name=name, iati_org_id=iati_org_id, internal_org_id=internal_org_id, event=ERROR_XML_PARSING, extra=e.message, ) if pk: ok, message, data = put_org(pk, internal_org_id, org_as_dict, user_cred) log(message, data) print message.format(**data) else: ok, message, data = post_org(internal_org_id, org_as_dict, user_cred) log(message, data) print message.format(**data) if ok: message, data = post_internal_id(user_cred, RAIN_ORG_ID, internal_org_id, data['pk']) log(message, data) print message.format(**data)
def upload_activities(argv): user = credentials_from_args(argv) if user: xml = load_xml(RAIN_IATI_ACTIVITES_URL) if xml: save_xml(xml, "rain_activities_{datetime}.xml") parser = etree.XMLParser(ns_clean=True, recover=True, encoding='utf-8') root = etree.fromstring(xml, parser=parser) document_akvo_ns = root.nsmap['akvo'] assert document_akvo_ns == AKVO_NS, "Akvo name space is incorrect in the IATI XML" activities = root.findall('iati-activity') activity_count = len(activities) for i, activity in enumerate(activities): activity = RainActivity(activity, RAIN_ACTIVITY_NS, AKVO_NS) internal_id = activity.internal_id() iati_id = activity.iati_id() try: assert iati_id is not None, "No IATI ID found, for activity number {} in the XML".format( i + 1) except AssertionError, e: message = "No IATI ID for activity number {extra}" data = dict( event=ERROR_MISSING_IATI_ID, extra=i, ) log(message, data) print message.format(**data) continue rsr_id = activity.rsr_id() print "({current} of {activity_count}) Processing activity {iati_id}".format( current=i + 1, activity_count=activity_count, iati_id=iati_id), if len(activity.tree.findall('participating-org')) > 0: try: rsr_id = identify_rsr_project(user, rsr_id, iati_id, internal_id) except AssertionError, e: message = "Error identifying RSR project: ID {pk}, IATI ID: {iati_id}, Error message: \n{extra}" data = dict( pk=rsr_id, iati_id=iati_id, event=ERROR_IDENTIFY_RSR_PROJECT, extra=e.message, ) log(message, data) print message.format(**data) continue # HACK! removing duplicates of RAIN as participating-org for dupe_rain in activity.tree.xpath( 'participating-org[@akvo:ref="0"]', namespaces={ 'akvo': 'http://akvo.org/iati-activities' }): dupe_rain.getparent().remove(dupe_rain) if rsr_id: ok, message, data = put_an_activity( activity.tree, rsr_id, user) log(message, data) print message.format(**data) else: ok, message, data = post_an_activity( activity.tree, user) log(message, data) print message.format(**data) else: message = "Iati-activity {iati_id} has no participating-orgs, aborting" data = dict( iati_id=iati_id, event=ERROR_NO_ORGS, ) log(message, data) print(message.format(**data))
def upload_organisations(argv): user_cred = credentials_from_args(argv) if user_cred: xml = load_xml(RAIN_ORGANISATIONS_URL) if xml: save_xml(xml, "rain_organisations_{datetime}.xml") parser = etree.XMLParser(ns_clean=True, recover=True, encoding='utf-8') root = etree.fromstring(xml, parser=parser) organisations = root.findall('organisation') for i, org_as_etree in enumerate(organisations): org_id = find_text( org_as_etree, 'internal_org_id@{{{}}}akvo_identifier'.format( RAIN_ORGANISATION_NS)) iati_org_id = find_text(org_as_etree, 'iati_org_id') internal_org_id = find_text(org_as_etree, 'internal_org_id') name = find_text(org_as_etree, 'name') assert org_id or ( iati_org_id or internal_org_id), \ "Cannot create a new organisation without either an IATI ID or an internal ID" print "Processing organisation {}".format(name), try: pk, content_owner_id = find_org(user_cred, RAIN_ORG_ID, org_id, iati_org_id, internal_org_id, name) except Exception, e: message = "Error trying to find organisation {name} ID {pk}, error message: {extra}" data = dict( pk=org_id, name=name, iati_org_id=iati_org_id, internal_org_id=internal_org_id, event=ERROR_UPLOAD_ORG, extra=e.message, ) log(message, data) print message.format(**data) continue if content_owner_id is not None and int( content_owner_id) != RAIN_ORG_ID: # Don't mess with orgs managed by others message = "Organisation {name} ID {pk} is managed by another organisation: {extra}" data = dict( pk=org_id, name=name, iati_org_id=iati_org_id, internal_org_id=internal_org_id, event=ERROR_OTHER_CONTENT_OWNER, extra=content_owner_id, ) log(message, data) print message.format(**data) continue try: org_as_dict = python_organisation(org_as_etree) except Exception, e: message = "Error converting organisation XML to json. Error message: {extra}" data = dict( pk=org_id, name=name, iati_org_id=iati_org_id, internal_org_id=internal_org_id, event=ERROR_XML_PARSING, extra=e.message, ) if pk: ok, message, data = put_org(pk, internal_org_id, org_as_dict, user_cred) log(message, data) print message.format(**data) else: ok, message, data = post_org(internal_org_id, org_as_dict, user_cred) log(message, data) print message.format(**data) if ok: message, data = post_internal_id(user_cred, RAIN_ORG_ID, internal_org_id, data['pk']) log(message, data) print message.format(**data)
internal_id=self.activity.internal_id(), iati_id=self.activity.iati_id(), event=ERROR_IMAGE_UPLOAD, extra=msg.format(internal_id=activity.internal_id()))) elif activity.iati_id(): try: self.project = Project.objects.get( iati_activity_id=activity.iati_id(), partnerships__organisation=RAIN_ORG_ID) return except: msg = "Could not find project or multiple projects found with IATI ID: {iati_id}" log( msg, dict(rsr_id=self.activity.rsr_id(), internal_id=self.activity.internal_id(), iati_id=self.activity.iati_id(), event=ERROR_IMAGE_UPLOAD, extra=msg.format(iati_id=activity.iati_id()))) raise Project.DoesNotExist def _current_image(self): image = ImageImporter(self.activity.current_image()) try: image.get_image() except Exception, e: log( "Error trying to fetch image to project. Image URL: {extra}", dict(rsr_id=self.activity.rsr_id(), internal_id=self.activity.internal_id(), iati_id=self.activity.iati_id(),