def set_publishing_status(publishing_statuses):
outsys("\nRunning {}() ".format(who_am_i()))
cordaid = Organisation.objects.get(pk=CORDAID_ORG_ID)
for internal_id, publish in publishing_statuses.items():
try:
status = PublishingStatus.objects.get(
project__partnerships__internal_id=internal_id,
project__partnerships__organisation=cordaid,
)
status.status = PublishingStatus.STATUS_PUBLISHED if publish else PublishingStatus.STATUS_UNPUBLISHED
status.save()
log(
u"Set publishing status for project ID: {pk}: {extra}",
dict(internal_id=internal_id,
pk=status.project.pk,
event=ACTION_PUBLISHING_SET,
extra=status.status))
outsys(".")
except Exception, e:
log(
u"Error setting publishing status for project {internal_id}\nException class: {extra}",
dict(internal_id=internal_id,
event=e.__class__,
extra=e.message),
)
outsys("*")
def update_organisation(org_etree, internal_org_id):
try:
org_dict = org_data_from_xml(org_etree)
referenced_org = internal_org_id.referenced_org
update_org = Organisation.objects.filter(pk=referenced_org.pk)
update_org.update(**org_dict)
log(
u"Updated organisation: {label}, Akvo ID: {pk}",
dict(
log_type=LOG_ORGANISATIONS,
internal_id=internal_org_id.identifier,
label=referenced_org.name,
pk=referenced_org.pk,
event=ACTION_UPDATE_ORG
)
)
# return the updated organisation record to be used in the following steps
return update_org[0]
except Exception, e:
log(
u"Error trying to update organisation with Cordaid ID {internal_id} ",
dict(
log_type=LOG_ORGANISATIONS,
internal_id=internal_org_id.identifier,
event=ERROR_EXCEPTION,
extra=e.message
)
)
def create_new_organisation(org_etree, internal_id):
try:
org_dict = org_data_from_xml(org_etree)
referenced_org = Organisation.objects.create(**org_dict)
log(
u"Created new organisation: {label}, Akvo ID: {pk}",
dict(
log_type=LOG_ORGANISATIONS,
internal_id=internal_id,
label=referenced_org.name,
pk=referenced_org.pk,
event=ACTION_CREATE_ORG
)
)
return referenced_org
except Exception, e:
log(
u"Error trying to create organisation with Cordaid ID {internal_id} ",
dict(
log_type=LOG_ORGANISATIONS,
internal_id=internal_id,
event=ERROR_EXCEPTION,
extra=e.message
)
)
def upload_organisations(argv):
user_cred = credentials_from_args(argv)
reporting_org_id = user_org(user_cred, **dict())
if user_cred:
with open(ORGANISATIONS_XML, 'r') as f:
root = etree.fromstring(f.read())
organisations = root.findall('object')
for i in range(len(organisations)):
internal_org_id = organisations[i].find('org_id').text
name = organisations[i].find('name').text
print "Processing organisation {name}".format(name=name),
if internal_org_id:
print " ID: {org_id}".format(org_id=internal_org_id),
import pdb
pdb.set_trace()
ok, pk = find_org(user_cred, reporting_org_id,
internal_org_id)
if pk:
ok, message, data = put_an_org(organisations[i],
user_cred, pk)
log(message, data)
print message.format(**data)
else:
ok, message, data = post_an_org(
organisations[i], user_cred)
log(message, data)
print message.format(**data)
if ok:
post_an_internal_id(user_cred, reporting_org_id,
internal_org_id, data['pk'])
else:
print "No internal org ID"
def upload_organisations(argv):
user_cred = credentials_from_args(argv)
reporting_org_id = user_org(user_cred, **dict())
if user_cred:
with open(ORGANISATIONS_XML, 'r') as f:
root = etree.fromstring(f.read())
organisations = root.findall('object')
for i in range(len(organisations)):
internal_org_id = organisations[i].find('org_id').text
name = organisations[i].find('name').text
print "Processing organisation {name}".format(name=name),
if internal_org_id:
print " ID: {org_id}".format(org_id=internal_org_id),
import pdb
pdb.set_trace()
ok, pk = find_org(user_cred, reporting_org_id,
internal_org_id)
if pk:
ok, message, data = put_an_org(organisations[i],
user_cred, pk)
log(message, data)
print message.format(**data)
else:
ok, message, data = post_an_org(
organisations[i], user_cred)
log(message, data)
print message.format(**data)
if ok:
post_an_internal_id(user_cred, reporting_org_id,
internal_org_id, data['pk'])
else:
print "No internal org ID"
def fix_funding(budgets):
"""
Add Cordaid as a funding partner to all its projects and "fill the project up"
"""
outsys("\nRunning {}() ".format(who_am_i()))
def assign_funding_partner(project, organisation, amount):
funding_partnership, created = Partnership.objects.get_or_create(
organisation=organisation,
project=project,
partner_type=Partnership.FUNDING_PARTNER,
defaults={'funding_amount': amount}
)
if created:
log(
u"Added {org_name} as funding partner to project {{pk}}, funding amount: {{extra}}".format(org_name=organisation.name),
dict(internal_id=internal_id, pk=project.pk, event=ACTION_FUNDING_SET, extra=amount)
)
else:
funding_partnership.funding_amount = amount
funding_partnership.save()
log(
u"Found {org_name} as funding partner to project {{pk}}, setting funding amount: {{extra}}".format(org_name=organisation.name),
dict(internal_id=internal_id, pk=project.pk, event=ACTION_FUNDING_FOUND, extra=amount)
)
cordaid = Organisation.objects.get(pk=CORDAID_ORG_ID)
others = Organisation.objects.get(pk=OTHERS_ORG_ID)
for budget in budgets:
internal_id = budget['internal_project_id']
try:
project = None
project = Project.objects.get(
partnerships__internal_id=internal_id, partnerships__organisation=cordaid
)
cordaid_funding = budget.get('cordaid_funding', 0)
others_funding = budget.get('others_funding', 0)
if cordaid_funding:
assign_funding_partner(project, cordaid, cordaid_funding)
if others_funding:
assign_funding_partner(project, others, others_funding)
total_budget = cordaid_funding + others_funding
old_budgets = BudgetItem.objects.filter(project=project)
old_budgets.delete()
BudgetItem.objects.create(
project=project,
label = BudgetItemLabel.objects.get(pk=13), #total budget label
amount = total_budget
)
log(
u"Total budget for project {pk}: {extra}",
dict(internal_id=internal_id, pk=project.pk, event=ACTION_BUDGET_SET, extra=total_budget)
)
outsys(".")
except Exception, e:
log(u"Error setting up funding partners for project {pk}\nException class: {extra}",
dict(internal_id=internal_id, pk=getattr(project, 'pk', None), event=e.__class__, extra=e.message),
)
outsys("*")
def create_cats_and_benches(business_units):
outsys("\nRunning {}() ".format(who_am_i()))
for internal_id, data in business_units.items():
for name in data['benchmarknames']:
outsys('.')
new_bench, created = Benchmarkname.objects.get_or_create(name=name)
if created:
log(u"Created bench: {id}, {name}", dict(id=new_bench.id, name=name))
else:
log(u"Found existing bench: {id}, {name}", dict(id=new_bench.id, name=name))
data['category'].benchmarknames.add(new_bench)
def upload_activities(argv):
user = credentials_from_args(argv)
if user:
with open(CORDAID_IATI_ACTIVITIES_XML, 'r') as f:
root = etree.fromstring(f.read())
document_akvo_ns = '{{{akvo_ns}}}'.format(
akvo_ns=root.nsmap['akvo'])
assert document_akvo_ns == AKVO_NS, "Akvo name space is incorrect in the IATI XML"
activities = root.findall('iati-activity')
activity_count = len(activities)
for i in range(activity_count):
internal_id = activities[i].get(AKVO_NS +
'internal-project-id')
iati_id = activities[i].findall('iati-identifier')[0].text
print "({current} of {activity_count}) Processing activity {iati_id}".format(
current=i + 1,
activity_count=activity_count,
iati_id=iati_id),
if len(activities[i].findall('participating-org')) > 0:
if internal_id:
ok, project = get_project_count(
user,
**dict(partnerships__internal_id=internal_id))
elif iati_id:
ok, project = get_project_count(
user,
**dict(partnerships__iati_activity_id=iati_id))
if not ok:
continue #error msg already output
project_count = project.response.json(
)['meta']['total_count']
if project_count == 0:
ok, message, data = post_an_activity(
activities[i], user)
log(message, data)
print message.format(**data)
elif project_count == 1:
pk = project.response.json()['objects'][0]['id']
ok, message, data = put_an_activity(
activities[i], pk, user)
log(message, data)
print message.format(**data)
elif project_count > 1:
data = dict(iati_id=iati_id,
event=ERROR_MULTIPLE_OBJECTS,
extra=internal_id)
log(None, data)
print(
"**** Error updating iati-activity: {iati_id}. "
"More than one project with internal ID {extra} exists."
.format(**data))
else:
message = "Iati-activity {iati_id} has no participating-orgs, aborting"
data = dict(
iati_id=iati_id,
event=ERROR_NO_ORGS,
)
log(message, data)
print(message.format(**data))
def find_cordaid_business_units(business_units):
outsys("\nRunning {}() ".format(who_am_i()))
known_business_units = {
"27239": dict(pk=CORDAID_ORG_ID),
"K6020": dict(pk=959, cat_name="Children & Education", fa="Education"),
"K6090": dict(pk=962, cat_name="Domestic", fa="Economic development"),
"K6030": dict(pk=961, cat_name="Disaster Recovery", fa="Economic development"),
"K6070": dict(pk=950, cat_name="Entrepreneurship", fa="Economic development"),
"K6110": dict(pk=1099, cat_name="Food Security", fa="Healthcare"),
"K6100": dict(pk=953, cat_name="Investments", fa="Economic development"),
"K6010": dict(pk=949, cat_name="Healthcare", fa="Healthcare"),
"K6060": dict(pk=1241, cat_name="Security & Justice", fa="Economic development"),
"K6080": dict(pk=946, cat_name="Urban Matters", fa="Economic development"),
"K6040": dict(pk=955, cat_name="Women's leadership", fa="Economic development"),
"K6050": dict(pk=960, cat_name="Extractives", fa="Economic development"),
}
cordaid = Organisation.objects.get(pk=CORDAID_ORG_ID)
for internal_id in business_units.keys():
cbu = known_business_units.get(internal_id, {'pk': -1})
pk, cat_name, fa_name = cbu['pk'], cbu.get('cat_name'), cbu.get('fa'),
try:
organisation = Organisation.objects.get(pk=pk)
outsys('.')
except:
outsys('*')
log(
u"No business unit with internal ID {internal_id}",
dict(
internal_id=internal_id,
event=ERROR_BUSINESS_UNIT_MISSING
)
)
continue
internal_org, created = InternalOrganisationID.objects.get_or_create(
recording_org=cordaid,
referenced_org=organisation,
identifier= internal_id
)
if cat_name:
new_cat, created = Category.objects.get_or_create(name=cat_name)
if created:
log(u"Created cat: {id}, {cat_name}",dict(id=new_cat.id, cat_name=cat_name))
new_cat.focus_area.add(FocusArea.objects.get(name=fa_name))
else:
log(u"Found existing cat: {id}, {cat_name}", dict(id=new_cat.id, cat_name=cat_name))
business_units.setdefault(internal_id, {'category': None, 'benchmarknames': []})['category'] = new_cat
cordaid.iati_org_id = CORDAID_IATI_ID
cordaid.save()
try:
dgis = Organisation.objects.get(pk=DGIS_ORG_ID)
dgis.iati_org_id = DGIS_IATI_ID
dgis.save()
except:
log(u"Can't find DGIS using ID {dgis_id}", dict(dgis_id=DGIS_ORG_ID),)
return business_units
def find_cordaid_business_units(business_units):
outsys("\nRunning {}() ".format(who_am_i()))
known_business_units = {
"27239": dict(pk=CORDAID_ORG_ID),
"K6020": dict(pk=959, cat_name="Children & Education", fa="Education"),
"K6090": dict(pk=962, cat_name="Domestic", fa="Economic development"),
"K6030": dict(pk=961, cat_name="Disaster Recovery", fa="Economic development"),
"K6070": dict(pk=950, cat_name="Entrepreneurship", fa="Economic development"),
"K6110": dict(pk=1099, cat_name="Food Security", fa="Healthcare"),
"K6100": dict(pk=953, cat_name="Investments", fa="Economic development"),
"K6010": dict(pk=949, cat_name="Healthcare", fa="Healthcare"),
"K6060": dict(pk=1241, cat_name="Security & Justice", fa="Economic development"),
"K6080": dict(pk=946, cat_name="Urban Matters", fa="Economic development"),
"K6040": dict(pk=955, cat_name="Women's leadership", fa="Economic development"),
"K6050": dict(pk=960, cat_name="Extractives", fa="Economic development"),
}
cordaid = Organisation.objects.get(pk=CORDAID_ORG_ID)
for internal_id in business_units.keys():
cbu = known_business_units.get(internal_id, {'pk': -1})
pk, cat_name, fa_name = cbu['pk'], cbu.get('cat_name'), cbu.get('fa'),
try:
organisation = Organisation.objects.get(pk=pk)
outsys('.')
except:
outsys('*')
log(
u"No business unit with internal ID {internal_id}",
dict(
internal_id=internal_id,
event=ERROR_BUSINESS_UNIT_MISSING
)
)
continue
internal_org, created = InternalOrganisationID.objects.get_or_create(
recording_org=cordaid,
referenced_org=organisation,
identifier=internal_id
)
if cat_name:
new_cat, created = Category.objects.get_or_create(name=cat_name)
if created:
log(u"Created cat: {id}, {cat_name}", dict(id=new_cat.id, cat_name=cat_name))
new_cat.focus_area.add(FocusArea.objects.get(name=fa_name))
else:
log(u"Found existing cat: {id}, {cat_name}", dict(id=new_cat.id, cat_name=cat_name))
business_units.setdefault(internal_id, {'category': None, 'benchmarknames': []})['category'] = new_cat
cordaid.iati_org_id = CORDAID_IATI_ID
cordaid.save()
try:
dgis = Organisation.objects.get(pk=DGIS_ORG_ID)
dgis.iati_org_id = DGIS_IATI_ID
dgis.save()
except:
log(u"Can't find DGIS using ID {dgis_id}", dict(dgis_id=DGIS_ORG_ID),)
return business_units
def assign_funding_partner(project, organisation, amount):
funding_partnership, created = Partnership.objects.get_or_create(
organisation=organisation,
project=project,
iati_organisation_role=Partnership.IATI_FUNDING_PARTNER,
defaults={'funding_amount': amount}
)
if created:
log(
"Added {org_name} as funding partner to project {{pk}}, funding amount: {{extra}}".format(org_name=organisation.name),
dict(internal_id=internal_id, pk=project.pk, event=ACTION_FUNDING_SET, extra=amount)
)
else:
funding_partnership.funding_amount = amount
funding_partnership.save()
log(
"Found {org_name} as funding partner to project {{pk}}, setting funding amount: {{extra}}".format(org_name=organisation.name),
dict(internal_id=internal_id, pk=project.pk, event=ACTION_FUNDING_FOUND, extra=amount)
)
def assign_funding_partner(project, organisation, amount):
funding_partnership, created = Partnership.objects.get_or_create(
organisation=organisation,
project=project,
iati_organisation_role=Partnership.IATI_FUNDING_PARTNER,
defaults={'funding_amount': amount}
)
if created:
log(
u"Added {org_name} as funding partner to project {{pk}}, funding amount: {{extra}}".format(org_name=organisation.name),
dict(internal_id=internal_id, pk=project.pk, event=ACTION_FUNDING_SET, extra=amount)
)
else:
funding_partnership.funding_amount = amount
funding_partnership.save()
log(
u"Found {org_name} as funding partner to project {{pk}}, setting funding amount: {{extra}}".format(org_name=organisation.name),
dict(internal_id=internal_id, pk=project.pk, event=ACTION_FUNDING_FOUND, extra=amount)
)
def create_cordaid_business_units(business_units):
outsys("\nRunning {}() ".format(who_am_i()))
business_units_info = [
dict(pk=CORDAID_ORG_ID, internal_id="27239"),
dict(pk=959, internal_id="K6020", cat_name="Children & Education", fa="Education"),
dict(pk=962, internal_id="K6090", cat_name="Domestic", fa="Economic development"),
dict(pk=961, internal_id="K6030", cat_name="Disaster Recovery", fa="Economic development"),
dict(pk=950, internal_id="K6070", cat_name="Entrepreneurship", fa="Economic development"),
dict(pk=1099, internal_id="K6110", cat_name="Food Security", fa="Healthcare"),
dict(pk=953, internal_id="K6100", cat_name="Investments", fa="Economic development"),
dict(pk=949, internal_id="K6010", cat_name="Healthcare", fa="Healthcare"),
dict(pk=1241, internal_id="K6060", cat_name="Security & Justice", fa="Economic development"),
dict(pk=946, internal_id="K6080", cat_name="Urban Matters", fa="Economic development"),
dict(pk=955, internal_id="K6040", cat_name="Women's leadership", fa="Economic development"),
dict(pk=960, internal_id="K6050", cat_name="Extractives", fa="Economic development"),
]
cordaid = Organisation.objects.get(pk=CORDAID_ORG_ID)
for data in business_units_info:
outsys('.')
pk, identifier = data['pk'], data['internal_id']
cat_name, fa_name = data.get('cat_name'), data.get('fa')
try:
organisation = Organisation.objects.get(pk=pk)
except:
log(
u"No business unit with id {pk}, internal ID {identifier}",
dict(pk=pk, identifier=identifier),
)
continue
internal_org, created = InternalOrganisationID.objects.get_or_create(
recording_org=cordaid,
referenced_org=organisation,
identifier= identifier
)
if cat_name:
new_cat, created = Category.objects.get_or_create(name=cat_name)
if created:
log(u"Created cat: {id}, {cat_name}",dict(id=new_cat.id, cat_name=cat_name))
new_cat.focus_area.add(FocusArea.objects.get(name=fa_name))
else:
log(u"Found existing cat: {id}, {cat_name}", dict(id=new_cat.id, cat_name=cat_name))
business_units.setdefault(identifier, {'category': None, 'benchmarknames': []})['category'] = new_cat
cordaid.iati_org_id = CORDAID_IATI_ID
cordaid.save()
try:
dgis = Organisation.objects.get(pk=DGIS_ORG_ID)
dgis.iati_org_id = DGIS_IATI_ID
dgis.save()
except:
log(u"Can't find DGIS using ID {dgis_id}", dict(dgis_id=DGIS_ORG_ID),)
return business_units
def import_images(image_dir, photos):
outsys("\nRunning {}() ".format(who_am_i()))
for image_name in os.listdir(image_dir):
photo_id, ext = splitext(image_name)
if ext.lower() in ['.png', '.jpg', '.jpeg', '.gif']:
try:
internal_id = photos.get(
photo_id,
{'internal_project_id': None})['internal_project_id']
project = Project.objects.get(
partnerships__internal_id=internal_id)
filename = model_and_instance_based_filename(
'Project', project.pk, 'current_image', image_name)
with open(os.path.join(image_dir, image_name), 'rb') as f:
image_data = f.read()
image_temp = NamedTemporaryFile(delete=True)
image_temp.write(image_data)
image_temp.flush()
project.current_image.save(filename,
File(image_temp),
save=True)
f.close()
project.current_image_caption = photos.get(
photo_id, {'image_caption': ''})['image_caption']
project.current_image_credit = photos.get(
photo_id, {'image_credit': ''})['image_credit']
project.save()
log(
u"Uploaded image to project {pk}",
dict(internal_id=internal_id,
pk=project.pk,
event=ACTION_SET_IMAGE))
outsys(".")
except Exception, e:
log(
u"Upload failed. internal_id: {internal_id} Exception class: {extra}",
dict(internal_id=internal_id,
event=ERROR_IMAGE_UPLOAD,
extra=e.__class__),
)
outsys("*")
def set_publishing_status(publishing_statuses):
outsys("\nRunning {}() ".format(who_am_i()))
cordaid = Organisation.objects.get(pk=CORDAID_ORG_ID)
for internal_id, publish in publishing_statuses.items():
try:
status = PublishingStatus.objects.get(
project__partnerships__internal_id=internal_id,
project__partnerships__organisation=cordaid,
)
status.status = PublishingStatus.STATUS_PUBLISHED if publish else PublishingStatus.STATUS_UNPUBLISHED
status.save()
log(
u"Set publishing status for project ID: {pk}: {extra}",
dict(internal_id=internal_id, pk=status.project.pk, event=ACTION_PUBLISHING_SET, extra=status.status)
)
outsys(".")
except Exception, e:
log(u"Error setting publishing status for project {internal_id}\nException class: {extra}",
dict(internal_id=internal_id, event=e.__class__, extra=e.message),
)
outsys("*")
def create_cats_and_benches(business_units):
outsys("\nRunning {}() ".format(who_am_i()))
for internal_id, data in business_units.items():
if data.get('category'):
for name in data['benchmarknames']:
outsys('.')
new_bench, created = Benchmarkname.objects.get_or_create(
name=name)
if created:
log(
u"Created bench: {pk}, {label}",
dict(label=name,
pk=new_bench.id,
event=ACTION_BENCH_CREATE))
else:
log(
u"Found existing bench: {pk}, {label}",
dict(label=name,
pk=new_bench.id,
event=ACTION_BENCH_FOUND))
try:
data['category'].benchmarknames.add(new_bench)
except:
# we shouldn't end up here since we're testing for the existance of the category above
pass
else:
outsys('*')
log(
u"No category set of business unit with internal ID {internal_id}",
dict(internal_id=internal_id, event=ERROR_CATEGORY_MISSING))
def import_images(image_dir, photos):
outsys("\nRunning {}() ".format(who_am_i()))
for image_name in os.listdir(image_dir):
photo_id, ext = splitext(image_name)
if ext.lower() in ['.png', '.jpg', '.jpeg', '.gif']:
try:
internal_id = photos.get(
photo_id, {'internal_project_id': None}
)['internal_project_id']
project = Project.objects.get(
partnerships__internal_id=internal_id
)
filename = model_and_instance_based_filename(
'Project', project.pk, 'current_image', image_name
)
with open(os.path.join(image_dir, image_name), 'rb') as f:
image_data = f.read()
image_temp = NamedTemporaryFile(delete=True)
image_temp.write(image_data)
image_temp.flush()
project.current_image.save(filename, File(image_temp), save=True)
f.close()
project.current_image_caption = photos.get(
photo_id, {'image_caption': ''}
)['image_caption']
project.current_image_credit = photos.get(
photo_id, {'image_credit': ''}
)['image_credit']
project.save()
log(
u"Uploaded image to project {pk}",
dict(internal_id=internal_id, pk=project.pk, event=ACTION_SET_IMAGE))
outsys(".")
except Exception, e:
log(
u"Upload failed. internal_id: {internal_id} Exception class: {extra}",
dict(internal_id=internal_id, event=ERROR_IMAGE_UPLOAD, extra=e.__class__),
)
outsys("*")
def organisation_logo(org_etree, internal_id, org):
logo_file = glob.glob(
os.path.join(
CORDAID_LOGOS_DIR,
u"{logo_id}.*".format(logo_id=text_from_xpath(org_etree, 'logo_id'))
)
)
if len(logo_file) == 1:
logo_filename = basename(logo_file[0])
_, extension = splitext(logo_filename)
if extension.lower() in (".png", ".jpg", ".jpeg", ".gif"):
filename = model_and_instance_based_filename(
"Organisation",
org.pk,
"logo",
logo_filename
)
with open(os.path.join(CORDAID_LOGOS_DIR, logo_filename), "rb") as f:
logo_data = f.read()
logo_tmp = NamedTemporaryFile(delete=True)
logo_tmp.write(logo_data)
logo_tmp.flush()
org.logo.save(
filename, File(logo_tmp), save=True
)
log(
u" Added logo {extra} to org {pk}, ",
dict(
log_type=LOG_ORGANISATIONS,
internal_id=internal_id,
pk=org.pk,
label=org.name,
event=ACTION_SET_IMAGE,
extra= filename,
)
)
def upload_activities(argv):
user = credentials_from_args(argv)
if user:
with open(CORDAID_IATI_ACTIVITIES_XML, 'r') as f:
root = etree.fromstring(f.read())
document_akvo_ns = '{akvo_ns}'.format(akvo_ns=root.nsmap['akvo'])
assert document_akvo_ns == AKVO_NS, "Akvo name space is incorrect in the IATI XML"
activities = root.findall('iati-activity')
activity_count = len(activities)
for i in range(activity_count):
internal_id = activities[i].get(AKVO_NS + 'internal-project-id')
iati_id=activities[i].findall('iati-identifier')[0].text
print "({current} of {activity_count}) Processing activity {iati_id}".format(current=i+1, activity_count=activity_count, iati_id=iati_id),
if len(activities[i].findall('participating-org')) > 0:
if internal_id:
ok, project = get_project_count(user, **dict(partnerships__internal_id=internal_id))
elif iati_id:
ok, project = get_project_count(user, **dict(iati_activity_id=iati_id))
if not ok:
continue #error msg already output
project_count = project.response.json()['meta']['total_count']
if project_count == 0:
ok, message, data = post_an_activity(activities[i], user)
log(message, data)
print message.format(**data)
elif project_count == 1:
pk = project.response.json()['objects'][0]['id']
ok, message, data = put_an_activity(activities[i], pk, user)
log(message, data)
print message.format(**data)
elif project_count > 1:
data = dict(iati_id=iati_id, event=ERROR_MULTIPLE_OBJECTS, extra=internal_id)
log(None, data)
print(
"**** Error updating iati-activity: {iati_id}. "
"More than one project with internal ID {extra} exists.".format(**data)
)
else:
message = "Iati-activity {iati_id} has no participating-orgs, aborting"
data = dict(iati_id = iati_id, event = ERROR_NO_ORGS,)
log(message, data)
print(message.format(**data))
def update_organisation(org_etree, internal_org_id, cordaid):
try:
org_dict = org_data_from_xml(org_etree)
referenced_org = internal_org_id.referenced_org
update_org = Organisation.objects.filter(pk=referenced_org.pk)
content_owner = update_org[0].content_owner
if content_owner and content_owner != cordaid:
log(
u"Organisation content owned by different organisation: {label}, Akvo ID: {pk}, owned by: {extra}",
dict(
log_type=LOG_ORGANISATIONS,
internal_id=internal_org_id.identifier,
label=referenced_org.name,
pk=referenced_org.pk,
event=OWNER_CONTENT,
extra=content_owner.name
)
)
# return None so that organisation does not get updated afterwards
return None
else:
update_org.update(**org_dict)
log(
u"Updated organisation: {label}, Akvo ID: {pk}",
dict(
log_type=LOG_ORGANISATIONS,
internal_id=internal_org_id.identifier,
label=referenced_org.name,
pk=referenced_org.pk,
event=ACTION_UPDATE_ORG
)
)
# return the updated organisation record to be used in the following steps
return update_org[0]
except Exception, e:
log(
u"Error trying to update organisation with Cordaid ID {internal_id} ",
dict(
log_type=LOG_ORGANISATIONS,
internal_id=internal_org_id.identifier,
event=ERROR_EXCEPTION,
extra=e.message
)
)
def set_location_for_org(org_etree, internal_id, org):
if not org.primary_location:
iso_code = text_from_xpath(org_etree, 'location/object/iso_code').lower()
if not iso_code == "ww!":
country = custom_get_or_create_country(iso_code)
location = OrganisationLocation.objects.create(
country=country,
location_target=org
)
org.locations.add(location)
org.primary_location = location
org.save()
log(
u" Added location to org {pk}",
dict(
log_type=LOG_ORGANISATIONS,
internal_id=internal_id,
pk=org.pk,
label=org.name,
event=ACTION_LOCATION_SET,
)
)
else:
log(
u"Couldn't create location for org {pk}, no proper country code",
dict(
log_type=LOG_ORGANISATIONS,
internal_id=internal_id,
pk=org.pk,
label=org.name,
event=ERROR_COUNTRY_CODE,
)
)
else:
log(
u" Org {pk} already has a location.",
dict(
log_type=LOG_ORGANISATIONS,
internal_id=internal_id,
pk=org.pk,
label=org.name,
event=ACTION_LOCATION_FOUND,
)
)
def create_cats_and_benches(business_units):
outsys("\nRunning {}() ".format(who_am_i()))
for internal_id, data in business_units.items():
if data.get('category'):
for name in data['benchmarknames']:
outsys('.')
new_bench, created = Benchmarkname.objects.get_or_create(name=name)
if created:
log(u"Created bench: {pk}, {label}",
dict(
label=name,
pk=new_bench.id,
event=ACTION_BENCH_CREATE
)
)
else:
log(u"Found existing bench: {pk}, {label}",
dict(
label=name,
pk=new_bench.id,
event=ACTION_BENCH_FOUND
)
)
try:
data['category'].benchmarknames.add(new_bench)
except:
# we shouldn't end up here since we're testing for the existance of the category above
pass
else:
outsys('*')
log(
u"No category set of business unit with internal ID {internal_id}",
dict(
internal_id=internal_id,
event=ERROR_CATEGORY_MISSING
)
)
with open(xml_file, "rb") as f:
root = etree.fromstring(f.read())
cordaid = Organisation.objects.get(id=CORDAID_ORG_ID)
for org_etree in root:
outsys('.')
internal_id = text_from_xpath(org_etree, 'org_id')
try:
internal_org_id = InternalOrganisationID.objects.get(
recording_org=cordaid,
identifier=internal_id
)
log(
u"Found existing org {label} (Akvo PK {pk}) with Cordaid internal ID '{internal_id}'",
dict(
log_type=LOG_ORGANISATIONS,
label=internal_org_id.referenced_org.name,
pk=internal_org_id.referenced_org.pk,
internal_id=internal_id,
event=ACTION_FOUND
)
)
referenced_org = update_organisation(org_etree, internal_org_id)
set_location_for_org(org_etree, internal_id, referenced_org)
except InternalOrganisationID.MultipleObjectsReturned:
log(
u"Error from lookup of internal ID {internal_id}. Multiple objects found.",
dict(
log_type=LOG_ORGANISATIONS,
internal_id=internal_id,
event=ERROR_MULTIPLE_OBJECTS
),
)
def fix_funding(budgets):
"""
Add Cordaid as a funding partner to all its projects and "fill the project up"
"""
outsys("\nRunning {}() ".format(who_am_i()))
def assign_funding_partner(project, organisation, amount):
funding_partnership, created = Partnership.objects.get_or_create(
organisation=organisation,
project=project,
iati_organisation_role=Partnership.IATI_FUNDING_PARTNER,
defaults={'funding_amount': amount}
)
if created:
log(
"Added {org_name} as funding partner to project {{pk}}, funding amount: {{extra}}".format(org_name=organisation.name),
dict(internal_id=internal_id, pk=project.pk, event=ACTION_FUNDING_SET, extra=amount)
)
else:
funding_partnership.funding_amount = amount
funding_partnership.save()
log(
"Found {org_name} as funding partner to project {{pk}}, setting funding amount: {{extra}}".format(org_name=organisation.name),
dict(internal_id=internal_id, pk=project.pk, event=ACTION_FUNDING_FOUND, extra=amount)
)
cordaid = Organisation.objects.get(pk=CORDAID_ORG_ID)
others = Organisation.objects.get(pk=OTHERS_ORG_ID)
for budget in budgets:
internal_id = budget['internal_project_id']
try:
project = None
project = Project.objects.get(
partnerships__internal_id=internal_id, partnerships__organisation=cordaid
)
project.set_reporting_org(cordaid)
project.save()
cordaid_funding = budget.get('cordaid_funding', 0)
others_funding = budget.get('others_funding', 0)
if cordaid_funding:
assign_funding_partner(project, cordaid, cordaid_funding)
if others_funding:
assign_funding_partner(project, others, others_funding)
total_budget = cordaid_funding + others_funding
old_budgets = BudgetItem.objects.filter(project=project)
old_budgets.delete()
BudgetItem.objects.create(
project=project,
label=BudgetItemLabel.objects.get(pk=BudgetItemLabel.TOTAL_BUDGET_LABEL_ID),
amount=total_budget
)
log(
"Total budget for project {pk}: {extra}",
dict(internal_id=internal_id, pk=project.pk, event=ACTION_BUDGET_SET, extra=total_budget)
)
outsys(".")
except Exception as e:
log("Error setting up funding partners for project {pk}\nException class: {extra}",
dict(internal_id=internal_id, pk=getattr(project, 'pk', None), event=e.__class__, extra=str(e)),
)
outsys("*")
outsys('\n')
