def setUp(self):
"""Setup."""
valid_host = "partner1.{}".format(settings.AKVOAPP_DOMAIN)
self.c = Client(HTTP_HOST=valid_host)
o1 = Organisation(name='p1', long_name='Partner1')
o1.save()
ps1 = PartnerSite(organisation=o1, hostname='partner1', cname='projects.partner1.org')
ps1.save()
iati_version = Version(code=settings.IATI_VERSION)
iati_version.save()
def setUp(self):
"""Setup."""
valid_host = "partner1.{}".format(settings.AKVOAPP_DOMAIN)
self.c = Client(HTTP_HOST=valid_host)
o1 = Organisation(name='p1', long_name='Partner1')
o1.save()
ps1 = PartnerSite(organisation=o1,
hostname='partner1',
cname='projects.partner1.org')
ps1.save()
def setUp(self):
"""Setup."""
# valid_host = "partner1.{}".format(settings.AKVOAPP_DOMAIN)
self.cname = "projects.partner1.org"
self.c = Client(HTTP_HOST=self.cname)
o1 = Organisation(name='p1', long_name='Partner1')
o1.save()
ps1 = PartnerSite(organisation=o1,
hostname='partner1',
cname=self.cname)
ps1.save()
iati_version = Version(code=settings.IATI_VERSION)
iati_version.save()
def organisation_data_from_etree(self, tree):
def find_text(tree, str):
element = tree.find(str)
if element is None:
return ''
return element.text.strip() if element.text else ""
def location_data(location_tree):
if location_tree is None:
return []
iso_code = find_text(location_tree, 'iso_code').lower()
country, created = Country.objects.get_or_create(**Country.fields_from_iso_code(iso_code))
country = country.id
latitude = find_text(location_tree, 'latitude') or 0
longitude = find_text(location_tree, 'longitude') or 0
primary = True
return [dict(latitude=latitude, longitude=longitude, country=country, primary=primary)]
#id = find_text(tree, 'org_id')
long_name = find_text(tree, 'name')
name = long_name[:25]
description = find_text(tree, 'description')
url = find_text(tree, 'url')
iati_type = find_text(tree, 'iati_organisation_type')
new_organisation_type = int(iati_type) if iati_type else 22
organisation_type = Organisation.org_type_from_iati_type(new_organisation_type)
locations = location_data(tree.find('location/object'))
return dict(
name=name, long_name=long_name, description=description, url=url,
organisation_type=organisation_type, new_organisation_type=new_organisation_type,
locations=locations
)
def _create_filters_query(request):
"""Returns a Q object expression based on query parameters."""
location_param = int_or_none(request.GET.get('location'))
organisation_param = int_or_none(request.GET.get('organisation'))
sector_param = int_or_none(request.GET.get('sector'))
title_or_subtitle_param = request.GET.get('title_or_subtitle')
location_filter = (get_m49_filter(location_param,
use_recipient_country=False)
if location_param else None)
organisation_filter = (Organisation.all_updates_filter(organisation_param)
if organisation_param else None)
sector_filter = (Q(project__sectors__sector_code=sector_param,
project__sectors__vocabulary='2')
if sector_param else None)
title_filter = Q(title__icontains=title_or_subtitle_param
) if title_or_subtitle_param else None
all_filters = [
location_filter,
organisation_filter,
sector_filter,
]
filters = [_f for _f in all_filters if _f]
return reduce(lambda x, y: x & y,
filters) if filters else None, title_filter
def _create_filters_query(request):
"""Returns a Q object expression based on query parameters."""
location_param = int_or_none(request.GET.get('location'))
organisation_param = int_or_none(request.GET.get('organisation'))
sector_param = int_or_none(request.GET.get('sector'))
title_or_subtitle_param = request.GET.get('title_or_subtitle')
location_filter = (
get_m49_filter(location_param, use_recipient_country=False) if location_param else None
)
organisation_filter = (
Organisation.all_updates_filter(organisation_param)
if organisation_param else None
)
sector_filter = (
Q(project__sectors__sector_code=sector_param, project__sectors__vocabulary='2')
if sector_param else None
)
title_filter = Q(title__icontains=title_or_subtitle_param) if title_or_subtitle_param else None
all_filters = [
location_filter,
organisation_filter,
sector_filter,
]
filters = filter(None, all_filters)
return reduce(lambda x, y: x & y, filters) if filters else None, title_filter
def organisation_data_from_etree(self, tree):
def find_text(tree, str):
element = tree.find(str)
if element is None:
return ''
return element.text.strip() if element.text else ""
def location_data(location_tree):
if location_tree is None:
return []
iso_code = find_text(location_tree, 'iso_code').lower()
country, created = Country.objects.get_or_create(**Country.fields_from_iso_code(iso_code))
country = country.id
latitude = find_text(location_tree, 'latitude') or 0
longitude = find_text(location_tree, 'longitude') or 0
primary = True
return [dict(latitude=latitude, longitude=longitude, country=country, primary=primary)]
#id = find_text(tree, 'org_id')
long_name = find_text(tree, 'name')
name = long_name[:25]
description = find_text(tree, 'description')
url = find_text(tree, 'url')
iati_type = find_text(tree, 'iati_organisation_type')
new_organisation_type = int(iati_type) if iati_type else 22
organisation_type = Organisation.org_type_from_iati_type(new_organisation_type)
locations = location_data(tree.find('location/object'))
return dict(
name=name, long_name=long_name, description=description, url=url,
organisation_type=organisation_type, new_organisation_type=new_organisation_type,
locations=locations
)
def python_organisation(tree):
"""
Create a python data structure representing an organisation from the etree representation
"""
def location_data(location_tree):
if location_tree is None:
return []
iso_code = find_text(location_tree, 'iso_code').lower()
if iso_code:
country, created = Country.objects.get_or_create(
**Country.fields_from_iso_code(iso_code))
country = country.id
else:
# TODO: error reporting
return []
latitude = find_text(location_tree, 'latitude') or 0
longitude = find_text(location_tree, 'longitude') or 0
primary = True
return [
dict(
latitude=latitude,
longitude=longitude,
country=country,
primary=primary,
)
]
def import_logo(tree):
logo = find_text(tree, 'logo')
if logo:
image = ImageImporter(logo)
image.get_image()
return image.to_base64()
return None
long_name = find_text(tree, 'name')
name = long_name[:25]
description = find_text(tree, 'description')
url = find_text(tree, 'url')
iati_type = find_text(tree, 'iati_organisation_type')
new_organisation_type = int(iati_type) if iati_type else 22
organisation_type = Organisation.org_type_from_iati_type(
new_organisation_type)
content_owner = RAIN_ORG_ID
locations = location_data(tree.find('locations/object'))
org_dict = dict(
name=name,
long_name=long_name,
description=description,
url=url,
organisation_type=organisation_type,
new_organisation_type=new_organisation_type,
content_owner=content_owner,
locations=locations,
)
logo = import_logo(tree)
if logo:
org_dict['logo'] = logo
return org_dict
def python_organisation(tree):
"""
Create a python data structure representing an organisation from the etree representation
"""
def location_data(location_tree):
if location_tree is None:
return []
iso_code = find_text(location_tree, 'iso_code').lower()
if iso_code:
country, created = Country.objects.get_or_create(**Country.fields_from_iso_code(iso_code))
country = country.id
else:
# TODO: error reporting
return []
latitude = find_text(location_tree, 'latitude') or 0
longitude = find_text(location_tree, 'longitude') or 0
primary = True
return [dict(
latitude=latitude,
longitude=longitude,
country=country,
primary=primary,
)]
def import_logo(tree):
logo = find_text(tree, 'logo')
if logo:
image = ImageImporter(logo)
image.get_image()
return image.to_base64()
return None
long_name = find_text(tree, 'name')
name = long_name[:25]
description = find_text(tree, 'description')
url = find_text(tree, 'url')
iati_type = find_text(tree, 'iati_organisation_type')
new_organisation_type = int(iati_type) if iati_type else 22
organisation_type = Organisation.org_type_from_iati_type(new_organisation_type)
content_owner = RAIN_ORG_ID
locations = location_data(tree.find('locations/object'))
org_dict = dict(
name=name,
long_name=long_name,
description=description,
url=url,
organisation_type=organisation_type,
new_organisation_type=new_organisation_type,
content_owner=content_owner,
locations=locations,
)
logo = import_logo(tree)
if logo:
org_dict['logo'] = logo
return org_dict
def org_data_from_xml(org_etree):
# keys are Organisation field names, values are xpath expressions for getting those values from the org_etree
xpaths = dict(
name='name',
description='description',
url='url',
new_organisation_type='iati_organisation_type',
)
# get the raw data from the org_etree
org_data = data_from_xpaths(xpaths, org_etree)
# transform data
org_data['long_name'] = org_data['name']
org_data['name'] = org_data['name'][:25]
org_data['organisation_type'] = Organisation.org_type_from_iati_type(int(org_data['new_organisation_type']))
return org_data
def organisation_data_from_etree(self, tree):
def find_text(tree, str):
element = tree.find(str)
if element is None:
return ''
return element.text.strip() if element.text else ""
long_name = find_text(tree, 'name')
name = long_name[:25]
description = find_text(tree, 'description')
url = find_text(tree, 'url')
iati_type = find_text(tree, 'iati_organisation_type')
new_organisation_type = int(iati_type) if iati_type else 22
organisation_type = Organisation.org_type_from_iati_type(new_organisation_type)
return dict(
name=name, long_name=long_name, description=description, url=url,
organisation_type=organisation_type, new_organisation_type=new_organisation_type,
)
def import_orgs(xml_file):
with open(xml_file, "rb") as f:
report_data = dict(created=0, failed=0, updated=0)
root = etree.fromstring(f.read())
for element in root:
recording_org = Organisation.objects.get(id=CORDAID_ORG_ID)
identifier = element.findtext("org_id")
try: # Find existing InternalOrganisationID & Organisation ...
action = "updated"
internal_org_id = InternalOrganisationID.objects.get(recording_org=recording_org, identifier=identifier)
referenced_org = internal_org_id.referenced_org
except: # ... or attempt to create new ones
try:
action = "created"
referenced_org = Organisation()
referenced_org.save()
internal_org_id = InternalOrganisationID(
recording_org=recording_org, referenced_org=referenced_org, identifier=identifier
)
internal_org_id.save()
except Exception, e:
action = "failed"
internal_org_id.delete()
referenced_org.delete()
print(u"*** UNABLE TO CREATE NEW ORGANISATION! ")
name = element.findtext("name")
referenced_org.name, referenced_org.long_name = name[:25], name
referenced_org.description = element.findtext("description") or "N/A"
referenced_org.url = normalize_url(element.findtext("url"))
referenced_org.new_organisation_type = int(element.findtext("iati_organisation_type"))
referenced_org.organisation_type = get_organisation_type(referenced_org.new_organisation_type)
report_data[action] += 1
referenced_org.save()
log_and_alert(
u"{action} organisation {org_id} "
"(Cordaid ID: {cordaid_org_id})".format(
action=action.upper(), cordaid_org_id=internal_org_id.identifier, org_id=referenced_org.id
)
)
run_import_report("organisation", report_data)
def import_orgs(xml_file):
with open(xml_file, "rb") as f:
report_data = dict(created=0, failed=0, updated=0)
root = etree.fromstring(f.read())
for element in root:
recording_org = Organisation.objects.get(id=CORDAID_ORG_ID)
identifier = element.findtext("org_id")
try: # Find existing InternalOrganisationID & Organisation ...
action = "updated"
internal_org_id = InternalOrganisationID.objects.get(
recording_org=recording_org, identifier=identifier)
referenced_org = internal_org_id.referenced_org
except: # ... or attempt to create new ones
try:
action = "created"
referenced_org = Organisation()
referenced_org.save()
internal_org_id = InternalOrganisationID(
recording_org=recording_org,
referenced_org=referenced_org,
identifier=identifier)
internal_org_id.save()
for partner_type in PartnerType.objects.all():
referenced_org.partner_types.add(partner_type)
except Exception, e:
action = "failed"
internal_org_id.delete()
referenced_org.delete()
print(u"*** UNABLE TO CREATE NEW ORGANISATION! "
"Reason: {message}.".format(e.message))
name = element.findtext("name")
referenced_org.name, referenced_org.long_name = name[:25], name
referenced_org.description = element.findtext(
"description") or "N/A"
referenced_org.url = normalize_url(element.findtext("url"))
referenced_org.new_organisation_type = int(
element.findtext("iati_organisation_type"))
referenced_org.organisation_type = get_organisation_type(
referenced_org.new_organisation_type)
report_data[action] += 1
referenced_org.save()
log_and_alert(u"{action} organisation {org_id} "
"(Cordaid ID: {cordaid_org_id})".format(
action=action.upper(),
cordaid_org_id=internal_org_id.identifier,
org_id=referenced_org.id))
run_import_report("organisation", report_data)
