def make_transaction(transaction, exchange_rates, transaction_types, activities_covid_transactions):
"""
Main function to build a transaction object
from the transaction XML.
"""
provider = get_org(transaction, provider=True)
receiver = get_org(transaction, provider=False)
return {
'iati_identifier': transaction.getparent().find("iati-identifier").text,
'reporting_org_text': check_encoding(get_narrative(transaction.getparent().find("reporting-org"))),
'reporting_org_ref': transaction.getparent().find("reporting-org").get("ref"),
'reporting_org_type': transaction.getparent().find("reporting-org").get("type"),
'value_USD': get_value_usd(transaction, exchange_rates, transaction.getparent().get('default-currency')),
'transaction_date': transaction.find('transaction-date').get("iso-date"),
'transaction_type_code': transaction.find('transaction-type').get("code"),
'transaction_type': transaction_types[transaction.find('transaction-type').get("code")],
'provider_text': provider.get('text'),
'provider_ref': provider.get('ref'),
'provider_type': provider.get('type'),
'receiver_text': receiver.get('text'),
'receiver_ref': receiver.get('ref'),
'receiver_type': receiver.get('type'),
'finance_type': get_finance_type(transaction),
'sectors': get_sectors(transaction),
'countries_regions': get_countries_regions(transaction),
'covid_relevant': check_covid_relevant(transaction, activities_covid_transactions),
'humanitarian': transaction.get('humanitarian') or transaction.getparent().get('humanitarian') or None,
'covid_matches': get_covid_matches(transaction)
}
def get_organisations(reporting_orgs):
"""
Take the list of reporting orgs and create a lookup
dict that can then be used in cases where the
provider-org or receiver-org only states the ref.
"""
for org in reporting_orgs:
if org.get("ref") not in ORGANISATIONS:
ORGANISATIONS[org.get("ref")] = check_encoding(get_narrative(org))
return ORGANISATIONS
def get_participating_organisations(participating_orgs):
"""
Returns a dictionary of participating orgs, with
each organisation role as the key, and the values
for that particular organisation role
being a list of organisations with that role.
"""
orgs = {"1": [], "2": [], "3": [], "4": []}
for org in participating_orgs:
orgs[org.get('role')].append({
'text': get_narrative(org),
'ref': org.get('ref')
})
return orgs
def get_org(transaction, provider=True):
"""
Get organisation data from an organisation object
(`participating-org`, `provider-org` or `receiver-org`).
"""
transaction_type = transaction.find("transaction-type").get("code")
provider_receiver = {True: 'provider', False: 'receiver'}[provider]
if transaction.find('{}-org'.format(provider_receiver)) is not None:
_text = get_org_name(
ref=transaction.find('{}-org'.format(provider_receiver)).get("ref"),
text=get_narrative(transaction.find('{}-org'.format(provider_receiver)))
)
_ref = transaction.find('{}-org'.format(provider_receiver)).get("ref")
_type = transaction.find('{}-org'.format(provider_receiver)).get("type")
if (_ref is not None) or (_text is not None):
return {
'text': _text or _ref,
'ref': _ref or _text,
'type': _type
}
role = {
True: TRANSACTION_TYPES_RULES[transaction_type]['provider'],
False: TRANSACTION_TYPES_RULES[transaction_type]['receiver']}[provider]
if ((role == "reporter")
or (provider==True and transaction_type in ['3', '4'])
or (provider==False and transaction_type in ['1', '11', '13'])):
_text = get_org_name(
transaction.getparent().find("reporting-org").get("ref")
)
_ref = transaction.getparent().find("reporting-org").get("ref")
_type = transaction.getparent().find("reporting-org").get("type")
if (_ref is not None) or (_text is not None):
return {
'text': _text or _ref,
'ref': _ref or _text,
'type': _type
}
orgs = transaction.getparent().findall("participating-org[@role='{}']".format(role))
if len(orgs) == 1:
_text = get_org_name(
orgs[0].get('ref'),
get_narrative_text(orgs[0])
)
_ref = orgs[0].get("ref")
_type = orgs[0].get("type")
return {
'text': _text or _ref,
'ref': _ref or _text,
'type': _type
}
elif len(orgs) > 1:
return {
'text': "MULTIPLE",
'ref': "MULTIPLE",
'type': "MULTIPLE"
}
return {
'text': "UNKNOWN",
'ref': "UNKNOWN",
'type': "UNKNOWN"
}
def process_activity(activity, exchange_rates):
"""
Main function which reads data from a single `activity`
lxml ElementTree and extracts only the information we
need for the visualisation.
"""
def _get_transaction_sums_commitments(activity, exchange_rates, covid=False):
"""
Get the sum of commitment transactions (optionally,
only those transactions tagged as COVID-19 related).
"""
if covid==True:
return get_transaction_sums(
activity.xpath("transaction[transaction-type/@code='2' or transaction-type/@code='11'][humanitarian-scope/@code='EP-2020-000012-001' or contains(description/narrative/text(), 'COVID-19')]"),
exchange_rates,
activity.get("default-currency"))
return get_transaction_sums(
# UNCHR uses transaction type 11 (incoming commitment), not 2 (commitment)
activity.xpath("transaction[transaction-type/@code='2' or transaction-type/@code='11']"),
exchange_rates,
activity.get("default-currency"))
def _get_transaction_sums_disbursements(activity, exchange_rates):
"""
Get the sum of disbursement transactions.
"""
return get_transaction_sums(
activity.xpath("transaction[transaction-type/@code='3' or transaction-type/@code='4']"),
exchange_rates,
activity.get("default-currency"))
def _get_country_region(activity, covid=False):
"""
Get the list of countries or regions (optionally,
only from those transactions tagged as COVID-19 related).
"""
if covid==True:
return get_country_region(
[],
[],
activity.xpath("transaction[transaction-type/@code='2' or transaction-type/@code='11'][recipient-country][humanitarian-scope/@code='EP-2020-000012-001' or contains(description/narrative/text(), 'COVID-19')]"),
activity.xpath("transaction[transaction-type/@code='2' or transaction-type/@code='11'][recipient-region][humanitarian-scope/@code='EP-2020-000012-001' or contains(description/narrative/text(), 'COVID-19')]"))
return get_country_region(
activity.xpath("recipient-country[@vocabulary='1']|recipient-country[not(@vocabulary)]"),
activity.xpath("recipient-region[@vocabulary='1']|recipient-region[not(@vocabulary)]"),
activity.xpath("transaction[transaction-type/@code='2' or transaction-type/@code='11'][recipient-country]"),
activity.xpath("transaction[transaction-type/@code='2' or transaction-type/@code='11'][recipient-region]"))
def _get_sectors(activity):
"""
Get the list of sectors.
"""
return get_sector(
activity.xpath("sector[@vocabulary='1']|sector[not(@vocabulary)]"),
activity.xpath("transaction[transaction-type/@code='2' or transaction-type/@code='11'][sector]")
)
def _get_humanitarian_development(activity, transaction):
"""
Add coding as to whether this activity is
humanitarian; development; a mix (humanitarian
/ development; or unspecified.)
"""
if (activity == '1') and (not '0' in transaction):
return "humanitarian"
elif (activity == '1') and ('0' in transaction):
return "humanitarian / development"
elif (activity == '0') and ('1' in transaction):
return "humanitarian / development"
elif (activity == '0') and (not '1' in transaction):
return "development"
else:
return "unspecified"
def _get_finance_types(activity):
"""
Get the list of finance types.
"""
return get_finance_type(
activity.find('default-finance-type'),
activity.xpath("transaction[transaction-type/@code='2' or transaction-type/@code='11'][finance-type]")
)
def _get_covid_matches(activity):
"""
Add coding regarding which parts of the IATI COVID-19
Publishing Guidance this activity corresponds to.
"""
matches = []
titles = ", ".join(activity.xpath("title/narrative/text()"))
descriptions = ", ".join(activity.xpath("description/narrative/text()"))
humanitarian_scopes = ", ".join(activity.xpath("humanitarian-scope/@code"))
tag = activity.xpath("tag/@code='COVID-19'")
transaction_descriptions = ", ".join(activity.xpath("transaction/description/narrative/text()"))
if ("COVID-19" in titles) or ("COVID 19" in titles): matches.append('title')
if ("COVID-19" in descriptions) or ("COVID 19" in descriptions): matches.append('description')
if ("EP-2020-000012-001" in humanitarian_scopes): matches.append('GLIDE')
if ("HCOVD20" in humanitarian_scopes): matches.append('HRP')
if tag: matches.append('tag')
if ("COVID-19" in transaction_descriptions) or ("COVID 19" in transaction_descriptions) or ("EP-2020-000012-001" in transaction_descriptions): matches.append('transaction-description')
return matches
data = {
"iatiIdentifier": activity.find('iati-identifier').text.strip(),
"title": get_narrative(activity.find('title')),
"reportingOrg": {
'text': get_narrative(activity.find('reporting-org')),
'ref': activity.find('reporting-org').get("ref"),
'type': activity.find('reporting-org').get("type")
},
"participatingOrganisation": get_participating_organisations(activity.findall('participating-org')),
"countriesRegions": _get_country_region(activity),
"countriesRegionsCOVID": _get_country_region(activity, True),
"sectors": _get_sectors(activity),
"financeTypes": _get_finance_types(activity),
"commitmentsUSD": _get_transaction_sums_commitments(activity, exchange_rates),
"disbursementsUSD": _get_transaction_sums_disbursements(activity, exchange_rates),
"budgetsUSD": get_budget_sums(activity.xpath("budget[@type='1']"), activity.xpath("budget[@type='2']"), exchange_rates, activity.get("default-currency")),
"commitmentsCOVID": _get_transaction_sums_commitments(activity, exchange_rates, True),
"locations": get_locations(activity.xpath("location")),
"COVIDComponent": bool(activity.xpath("transaction[humanitarian-scope/@code='EP-2020-000012-001' or contains(description/narrative/text(), 'COVID-19')]")),
"humanitarianDevelopment": _get_humanitarian_development(activity.get('humanitarian'), set(activity.xpath('transaction/@humanitarian'))),
"COVIDMatches": _get_covid_matches(activity)
}
return data