def guess_number(self, model_name, number_string):
# first strip non numeric values, except for -.,
decimal_string = re.sub(r'[^\d.,-]+', '', number_string)
try:
return Decimal(decimal_string)
except ValueError:
raise FieldValidationError(model_name, "value",
"Must be decimal or integer string")
except InvalidOperation:
raise FieldValidationError(model_name, "value",
"Must be decimal or integer string")
def iati_activities__iati_activity__location__gazetteer_entry(
self, element):
gazetteer_ref_code = element.attrib.get('gazetteer-ref')
gazetteer_ref = self.gazetteer_agency_mapping.get(gazetteer_ref_code)
code = element.text
if not gazetteer_ref_code:
raise RequiredFieldError(
"location/gazetteer-entry",
"gazetteer-ref",
"required attribute missing")
if not gazetteer_ref:
raise FieldValidationError(
"location/gazetteer-entry",
"gazetteer-ref",
"not found on the accompanying code list")
if not code:
raise RequiredFieldError(
"location/gazetteer-entry",
"text",
"required element empty")
location_id = E('location-id', code=code, vocabulary=gazetteer_ref)
super(
Parse, self
).iati_activities__iati_activity__location__location_id(
location_id
)
return element
def iati_organisations__iati_organisation__recipient_region_budget__value(
self, element):
"""atributes:
currency:USD
value-date:2014-01-01
tag:value"""
model = self.get_model('RecipientRegionBudget')
model.currency = self.get_or_none(codelist_models.Currency,
code=self._get_currency_or_raise(
'recipient-region-budget/value',
element.attrib.get('currency')))
value_date = element.attrib.get('value-date')
if value_date is None:
raise RequiredFieldError("RecipientRegionBudget", "value-date",
"required field missing.")
value_date = self.validate_date(value_date)
if not value_date:
raise FieldValidationError(
"RecipientRegionBudget",
"value-date",
"not in the correct range.",
None,
None,
)
model.value = element.text
model.value_date = value_date
# store element
return element
def iati_organisations__iati_organisation__total_expenditure__expense_line__value( # NOQA: E501
self, element):
"""
"""
model = self.get_model('TotalExpenditureBudgetLine',
self.total_expenditure_line_current_index)
currency = element.attrib.get("currency")
if not currency:
currency = getattr(self.get_model("Organisation"),
"default_currency")
if not currency:
raise RequiredFieldError(
"TotalExpenditureLine", "currency",
"must specify default-currency on iati-activity or as "
"currency on the element itself.")
else:
currency = self.get_or_none(codelist_models.Currency,
code=currency)
if currency is None:
raise FieldValidationError(
"TotalExpenditureLine",
"currency",
"not found on the accompanying codelist.",
None,
None,
)
model.value_date = self.validate_date(element.attrib.get('value-date'))
model.currency = currency
model.value = element.text
return element
def document_link_recipient_country(
document_link,
recipient_country_code,
):
warnings = []
errors = []
recipient_country = get_or_none(models.Country,
code=recipient_country_code)
if not recipient_country_code:
errors.append(
RequiredFieldError(
"organisation/document-link/recipient_country",
"code",
apiField="recipient_country.code",
))
elif not recipient_country:
errors.append(
FieldValidationError(
"organisation/document-link/recipient_country",
"code",
"recipient_country not found for code {}".format(
recipient_country_code),
apiField="recipient_country.code",
))
return {
"warnings": warnings,
"errors": errors,
"validated_data": {
"document_link": document_link,
"recipient_country": recipient_country,
},
}
def document_link_language(
document_link,
language_code,
):
warnings = []
errors = []
language = get_or_none(models.Language, code=language_code)
if not language_code:
errors.append(
RequiredFieldError(
"organisation/document-link/language",
"code",
apiField="language.code",
))
elif not language:
errors.append(
FieldValidationError(
"organisation/document-link/language",
"code",
"language not found for code {}".format(language_code),
apiField="language.code",
))
return {
"warnings": warnings,
"errors": errors,
"validated_data": {
"document_link": document_link,
"language": language,
},
}
def document_link_category(
document_link,
category_code,
):
warnings = []
errors = []
category = get_or_none(models.DocumentCategory, code=category_code)
if not category_code:
errors.append(
RequiredFieldError(
"organisation/document-link/category",
"code",
apiField="category.code",
))
elif not category:
errors.append(
FieldValidationError(
"organisation/document-link/category",
"code",
"category not found for code {}".format(category_code),
apiField="category.code",
))
return {
"warnings": warnings,
"errors": errors,
"validated_data": {
"document_link": document_link,
"category": category,
},
}
def iati_organisations__iati_organisation__document_link__document_date(
self, element):
"""attributes:
format:application/vnd.oasis.opendocument.text
url:http:www.example.org/docs/report_en.odt
tag:document-link"""
iso_date = element.attrib.get('iso-date')
if not iso_date:
raise RequiredFieldError("document-link/document-date", "iso-date",
"required attribute missing")
iso_date = self.validate_date(iso_date)
if not iso_date:
raise FieldValidationError("document-link/document-date",
"iso-date",
"iso-date not of type xsd:date")
document_link = self.get_model(
'OrganisationDocumentLink',
self.organisation_document_link_current_index)
document_link.iso_date = iso_date
return element
def organisation(
organisation_identifier,
default_lang,
default_currency,
name={},
iati_standard_version="2.02",
published=False,
):
warnings = []
errors = []
default_currency = get_or_none(models.Currency, pk=default_currency)
iati_standard_version = get_or_none(models.Version,
pk=iati_standard_version)
default_lang = get_or_none(models.Language, pk=default_lang)
if not default_lang:
warnings.append(
RequiredFieldError(
"organisation",
"default-lang",
apiField="default_lang",
))
if not validate_organisation_identifier(organisation_identifier):
errors.append(
FieldValidationError(
"organisation",
"organisation-identifier",
apiField="organisation_identifier",
))
name_narratives = name.get('narratives', [])
name_narratives = narratives_validate(name_narratives, default_lang, None,
warnings, errors, "name")
errors = errors + name_narratives['errors']
warnings = warnings + name_narratives['warnings']
return {
"warnings": warnings,
"errors": errors,
"validated_data": {
"organisation_identifier": organisation_identifier,
"normalized_organisation_identifier": organisation_identifier,
"default_lang": default_lang,
"default_currency": default_currency,
"iati_standard_version": iati_standard_version,
"published": published,
"name": {},
"name_narratives": name_narratives['validated_data'],
},
}
def iati_organisations__iati_organisation__reporting_org(self, element):
# Although OrganisationReportingOrganisation and Organisation has
# One-to-One relation on the database level, we check here whether
# element 'reporting-org' occurs only once in the parent element
# 'organisation'.
organisation = self.get_model('Organisation')
if 'OrganisationReportingOrganisation' in self.model_store:
for reporting_org in self.model_store[
'OrganisationReportingOrganisation']:
if reporting_org.organisation == organisation:
raise ParserError("Organisation",
"OrganisationReportingOrganisation",
"must occur no more than once.")
# narrative is parsed in different method but as it is required
# sub-element in 'name' element so we check it here.
narrative = element.xpath("narrative")
if len(narrative) < 1:
raise RequiredFieldError("OrganisationName", "narrative",
"must occur at least once.")
reporting_org_identifier = element.attrib.get("ref")
if reporting_org_identifier is None:
raise RequiredFieldError("OrganisationReportingOrganisation",
"ref", "required field missing.")
org_type = element.attrib.get("type")
if org_type is None:
raise RequiredFieldError("OrganisationReportingOrganisation",
"type", "required field missing.")
# here org_type is OrganisationType object.
org_type = self.get_or_none(codelist_models.OrganisationType,
code=org_type)
if org_type is None:
raise FieldValidationError(
"OrganisationReportingOrganisation",
"type",
"not found on the accompanying codelist.",
None,
None,
)
secondary_reporter = self.makeBool(
element.attrib.get("secondary-reporter"))
reporting_org = OrganisationReportingOrganisation()
reporting_org.organisation = organisation
reporting_org.org_type = org_type
reporting_org.secondary_reporter = secondary_reporter
reporting_org.reporting_org_identifier = reporting_org_identifier
self.register_model("OrganisationReportingOrganisation", reporting_org)
return element
def iati_organisations__iati_organisation__name(self, element):
name_list = self.get_model_list('OrganisationName')
if name_list and len(name_list) > 0:
raise FieldValidationError("name",
"Duplicate names are not allowed")
organisation = self.get_model('Organisation')
name = OrganisationName()
name.organisation = organisation
self.register_model('OrganisationName', name)
return element
def iati_organisations__iati_organisation__total_expenditure__value(
self, element):
"""
"""
model = self.get_model('TotalExpenditure',
self.total_expenditure_current_index)
code = element.attrib.get('currency')
currency = self.get_or_none(codelist_models.Currency, code=code)
if not code:
raise RequiredFieldError("total-expenditure/value/currency",
"code", "required element empty")
if not currency:
raise FieldValidationError(
"total-expenditure/value/currency", "code",
"not found on the accompanying code list")
model.value_date = self.validate_date(element.attrib.get('value-date'))
model.currency = currency
model.value = element.text
return element
def iati_activities__iati_activity__other_identifier(self, element):
"""atributes:
ref:ABC123-XYZ
owner-name:A1
tag:other-identifier"""
identifier = element.text
owner_ref = element.attrib.get('owner-ref')
owner_name = element.attrib.get('owner-name')
if not identifier:
raise RequiredFieldError("other-identifier", "text",
"required element empty")
if identifier and len(identifier) > 200:
raise FieldValidationError(
"other-identifier", "text",
"identifier is longer than 200 characters (unlikely and is \
most often a data bug)")
if not (owner_ref or owner_name):
raise RequiredFieldError(
"other-identifier", "owner-ref/owner-name",
"either owner_ref or owner_name must be set")
activity = self.get_model('Activity')
other_identifier = models.OtherIdentifier()
other_identifier.activity = activity
other_identifier.identifier = identifier
other_identifier.owner_ref = owner_ref
# TODO: refactor this to not create an lxml element
self.register_model('OtherIdentifier', other_identifier)
if element.text:
self.add_narrative(E('elem', owner_name), other_identifier)
return element
def iati_activities__iati_activity__participating_org(self, element):
"""atributes:
ref:BB-BBB-123456789
role:Funding
type:40
tag:participating-org"""
role_name = element.attrib.get('role')
role = self.get_or_none(codelist_models.OrganisationRole,
name=role_name)
if not role_name:
raise RequiredFieldError("participating-org", "role",
"required attribute missing")
if not role:
raise FieldValidationError(
"participating-org", "role",
"not found on the accompanying code list")
element.attrib['role'] = role.code
super(Parse,
self).iati_activities__iati_activity__participating_org(element)
participating_organisation = self.get_model(
'ActivityParticipatingOrganisation')
if element.text:
self.add_narrative(element, participating_organisation)
# workaround for IATI ref uniqueness limitation
participating_organisation.primary_name = self.get_primary_name(
element, participating_organisation.primary_name)
return element
def organisation_total_expenditure(
organisation,
period_start_raw,
period_end_raw,
value,
currency_code,
value_date_raw,
):
warnings = []
errors = []
currency = get_or_none(models.Currency, code=currency_code)
if not period_start_raw:
errors.append(
RequiredFieldError(
"recipient-org-budget",
"period-start",
apiField="period_start",
))
period_start = None
else:
try:
period_start = validate_date(period_start_raw)
except RequiredFieldError:
errors.append(
FieldValidationError(
"recipient-org-budget",
"period-start",
"iso-date not of type xsd:date",
apiField="period_start",
))
period_start = None
if not period_end_raw:
errors.append(
RequiredFieldError(
"recipient-org-budget",
"period-end",
apiField="period_end",
))
period_end = None
else:
try:
period_end = validate_date(period_end_raw)
except RequiredFieldError:
errors.append(
FieldValidationError(
"recipient-org-budget",
"period-end",
"iso-date not of type xsd:date",
apiField="period_end",
))
period_end = None
if not value:
errors.append(
RequiredFieldError(
"recipient-org-budget",
"value",
apiField="value",
))
if not value_date_raw:
errors.append(
RequiredFieldError(
"recipient-org-budget",
"value-date",
apiField="value_date",
))
value_date = None
else:
try:
value_date = validate_date(value_date_raw)
except RequiredFieldError:
errors.append(
FieldValidationError(
"recipient-org-budget",
"value-date",
"iso-date not of type xsd:date",
apiField="value_date",
))
value_date = None
if not currency and not organisation.default_currency:
errors.append(
RequiredFieldError(
"recipient-org-budget",
"currency",
"currency not specified and no default specified on \
organisation",
apiField="currency.code",
))
return {
"warnings": warnings,
"errors": errors,
"validated_data": {
"organisation": organisation,
"period_start": period_start,
"period_end": period_end,
"value": value,
"currency": currency,
"value_date": value_date,
},
}
def organisation_recipient_org_budget(
organisation,
status_code,
recipient_org_identifier,
period_start_raw,
period_end_raw,
value,
currency_code,
value_date_raw,
):
warnings = []
errors = []
status = get_or_none(models.BudgetStatus, code=status_code)
currency = get_or_none(models.Currency, code=currency_code)
recipient_org = get_or_none(organisation_models.Organisation,
pk=recipient_org_identifier)
if not status_code:
errors.append(
RequiredFieldError(
"recipient-org-budget",
"status",
apiField="status.code",
))
if not status:
errors.append(
FieldValidationError(
"recipient-org-budget",
"status",
"codelist entry not found for {}".format(status_code),
apiField="status.code",
))
if not recipient_org_identifier:
errors.append(
RequiredFieldError(
"recipient-org-identifier",
"recipient_org",
apiField="recipient_org.code",
))
# TODO: require that the org is in iati? - 2017-03-14
# if not recipient_org:
# errors.append(
# FieldValidationError(
# "recipient-org-budget",
# "recipient_org",
# "codelist entry not found for {}".format(
# recipient_org_identifier
# ),
# apiField="recipient_org.code",
# ))
if not period_start_raw:
errors.append(
RequiredFieldError(
"recipient-org-budget",
"period-start",
apiField="period_start",
))
period_start = None
else:
try:
period_start = validate_date(period_start_raw)
except RequiredFieldError:
errors.append(
FieldValidationError(
"recipient-org-budget",
"period-start",
"iso-date not of type xsd:date",
apiField="period_start",
))
period_start = None
if not period_end_raw:
errors.append(
RequiredFieldError(
"recipient-org-budget",
"period-end",
apiField="period_end",
))
period_end = None
else:
try:
period_end = validate_date(period_end_raw)
except RequiredFieldError:
errors.append(
FieldValidationError(
"recipient-org-budget",
"period-end",
"iso-date not of type xsd:date",
apiField="period_end",
))
period_end = None
if not value:
errors.append(
RequiredFieldError(
"recipient-org-budget",
"value",
apiField="value",
))
if not value_date_raw:
errors.append(
RequiredFieldError(
"recipient-org-budget",
"value-date",
apiField="value_date",
))
value_date = None
else:
try:
value_date = validate_date(value_date_raw)
except RequiredFieldError:
errors.append(
FieldValidationError(
"recipient-org-budget",
"value-date",
"iso-date not of type xsd:date",
apiField="value_date",
))
value_date = None
if not currency and not organisation.default_currency:
errors.append(
RequiredFieldError(
"recipient-org-budget",
"currency",
"currency not specified and no default specified on \
organisation",
apiField="currency.code",
))
return {
"warnings": warnings,
"errors": errors,
"validated_data": {
"organisation": organisation,
"status": status,
"recipient_org_identifier": recipient_org_identifier,
"recipient_org": recipient_org,
"period_start": period_start,
"period_end": period_end,
"value": value,
"currency": currency,
"value_date": value_date,
},
}
def organisation_recipient_region_budget(
organisation,
status_code,
region_code,
period_start_raw,
period_end_raw,
value,
currency_code,
value_date_raw,
):
warnings = []
errors = []
status = get_or_none(models.BudgetStatus, code=status_code)
currency = get_or_none(models.Currency, code=currency_code)
region = get_or_none(models.Region, code=region_code)
if not status_code:
errors.append(
RequiredFieldError(
"recipient-org-budget",
"status",
apiField="status.code",
))
if not status:
errors.append(
FieldValidationError(
"recipient-org-budget",
"status",
"codelist entry not found for {}".format(status_code),
apiField="status.code",
))
if not region_code:
errors.append(
RequiredFieldError(
"recipient-org-budget",
"region",
apiField="region.code",
))
if not region:
errors.append(
FieldValidationError(
"recipient-org-budget",
"region",
"codelist entry not found for {}".format(region_code),
apiField="region.code",
))
if not period_start_raw:
errors.append(
RequiredFieldError(
"recipient-org-budget",
"period-start",
apiField="period_start",
))
period_start = None
else:
try:
period_start = validate_date(period_start_raw)
except RequiredFieldError:
errors.append(
FieldValidationError(
"recipient-org-budget",
"period-start",
"iso-date not of type xsd:date",
apiField="period_start",
))
period_start = None
if not period_end_raw:
errors.append(
RequiredFieldError(
"recipient-org-budget",
"period-end",
apiField="period_end",
))
period_end = None
else:
try:
period_end = validate_date(period_end_raw)
except RequiredFieldError:
errors.append(
FieldValidationError(
"recipient-org-budget",
"period-end",
"iso-date not of type xsd:date",
apiField="period_end",
))
period_end = None
if not value:
errors.append(
RequiredFieldError(
"recipient-org-budget",
"value",
apiField="value",
))
if not value_date_raw:
errors.append(
RequiredFieldError(
"recipient-org-budget",
"value-date",
apiField="value_date",
))
value_date = None
else:
try:
value_date = validate_date(value_date_raw)
except RequiredFieldError:
errors.append(
FieldValidationError(
"recipient-org-budget",
"value-date",
"iso-date not of type xsd:date",
apiField="value_date",
))
value_date = None
if not currency and not organisation.default_currency:
errors.append(
RequiredFieldError(
"recipient-org-budget",
"currency",
"currency not specified and no default specified on \
organisation",
apiField="currency.code",
))
return {
"warnings": warnings,
"errors": errors,
"validated_data": {
"organisation": organisation,
"status": status,
"region": region,
"period_start": period_start,
"period_end": period_end,
"value": value,
"currency": currency,
"value_date": value_date,
},
}
def organisation_recipient_country_budget_line(
recipient_country_budget,
ref,
value,
currency_code,
value_date_raw,
narratives_data,
):
warnings = []
errors = []
if not narratives_data:
narratives_data = []
currency = get_or_none(models.Currency, code=currency_code)
if not value:
errors.append(
RequiredFieldError(
"total-budget/budget-line",
"value",
apiField="value",
))
if not value_date_raw:
errors.append(
RequiredFieldError(
"total-budget/budget-line",
"value-date",
apiField="value_date",
))
value_date = None
else:
try:
value_date = validate_date(value_date_raw)
except RequiredFieldError:
errors.append(
FieldValidationError(
"total-budget/budget-line",
"value-date",
"iso-date not of type xsd:date",
apiField="value_date",
))
value_date = None
if (not currency
and not recipient_country_budget.organisation.default_currency):
errors.append(
RequiredFieldError(
"total-budget/budget-line",
"currency",
"currency not specified and no default specified on \
organisation",
apiField="currency.code",
))
narratives = narratives_validate(
narratives_data, recipient_country_budget.organisation.default_lang,
recipient_country_budget.organisation.id, warnings, errors,
"total-budget/budget-line")
errors = errors + narratives['errors']
warnings = warnings + narratives['warnings']
return {
"warnings": warnings,
"errors": errors,
"validated_data": {
"recipient_country_budget": recipient_country_budget,
"ref": ref,
"value": value,
"currency": currency,
"value_date": value_date,
"narratives": narratives['validated_data'],
},
}
def organisation_document_link(
organisation,
url,
file_format_code,
document_date_raw,
title_narratives_data,
):
warnings = []
errors = []
if not title_narratives_data:
title_narratives_data = []
file_format = get_or_none(models.FileFormat, code=file_format_code)
if not file_format_code:
errors.append(
RequiredFieldError(
"organisation/document-link",
"file-format",
apiField="file_format.code",
))
elif not file_format:
errors.append(
FieldValidationError(
"organisation/document-link",
"file-format",
"format not found for code {}".format(file_format_code),
apiField="file_format.code",
))
# TODO: check the url actually resolves? - 2016-12-14
if not url:
errors.append(
RequiredFieldError(
"document_link",
"url",
apiField="url",
))
try:
document_date = validate_date(document_date_raw)
except RequiredFieldError:
# if not document_date:
# errors.append(
# RequiredFieldError(
# "organisation/document-link",
# "document-date",
# ))
document_date = None
title_narratives = narratives_validate(title_narratives_data,
organisation.default_lang,
organisation.id, warnings, errors,
"title")
errors = errors + title_narratives['errors']
warnings = warnings + title_narratives['warnings']
return {
"warnings": warnings,
"errors": errors,
"validated_data": {
"organisation": organisation,
"url": url,
"file_format": file_format,
"title_narratives": title_narratives['validated_data'],
"iso_date": document_date,
},
}