def add_code_list_item(self, elem):
tag = elem.tag
item = None
code = self.return_first(elem.xpath('code/text()'))
name = self.return_first(elem.xpath('name/text()'))
description = self.return_first(elem.xpath('description/text()')) or ''
language_name = self.return_first(elem.xpath('language/text()'))
category = self.return_first(elem.xpath('category/text()'))
url = self.return_first(elem.xpath('url/text()')) or ' '
model_name = tag
if tag == "Country":
name = name.lower().capitalize()
item = Country(language=language_name, data_source="IATI")
elif tag == "Region":
region_voc = RegionVocabulary.objects.get(code=1)
item = Region(region_vocabulary=region_voc)
if name is None or name == '':
logger.log(0, 'name is null in ' + tag)
name = code
model = None
try:
# to do; change app_label to iati_codelist after codelist app change
model = apps.get_model(app_label='geodata', model_name=model_name)
except LookupError:
pass
if not model:
try:
model = apps.get_model(app_label='geodata',
model_name=model_name)
except LookupError:
print(''.join(['Model not found: ', model_name]))
return False
if not item:
item = model()
item.code = code
item.name = name
item.codelist_iati_version = self.looping_through_version
item = self.add_to_model_if_field_exists(model, item, 'description',
description)
item = self.add_to_model_if_field_exists(model, item, 'url', url)
if category:
item = self.add_to_model_if_field_exists(model, item,
'category_id', category)
if item is not None and not model.objects.filter(
pk=item.code).exists():
try:
item.save()
except IntegrityError as err:
print("Error: {}".format(err))
pass
def update_polygon(self):
admin_countries = self.get_json_data(
"/../data_backup/allcountrycodes.json")
for k in admin_countries: # .get('features'):
country_iso2 = k.get('alpha-2').lower()
name = k.get('name').lower()
country_iso3 = k.get('alpha-3').lower()
if not country_iso2:
continue
c, created = Country.objects.get_or_create(iso2=country_iso2,
iso3=country_iso3,
name=name,
primary_name=True)
if created:
c.save()
# Add or update geolocation
try:
geolocation = Geolocation.objects.get(tag=c.name)
except Geolocation.DoesNotExist:
geolocation = Geolocation(content_object=c,
tag=c.name,
type='country')
geolocation.save()
print('Country : {name}:'.format(name=c.name))
poly_countries = self.get_json_data(
"/../data_backup/country_data.json").get('features')
for k in poly_countries: # .get('features'):
if 'iso2' in k.get('properties'):
iso2 = k.get('properties').get('iso2').lower()
filtered_set = Country.objects.filter(iso2=iso2,
primary_name=True)
if len(filtered_set) > 0:
c = filtered_set[0]
else:
name = k.get('properties').get('name').lower()
c = Country(name=name, iso2=iso2, primary_name=True)
c.save()
c.polygons = json.dumps(k.get('geometry'))
c.save()
# Add or update geolocation
try:
geolocation = Geolocation.objects.get(tag=c.name)
except Geolocation.DoesNotExist:
geolocation = Geolocation(content_object=c,
tag=c.name,
type='country')
geolocation.save()
print('Polygon : {name}:'.format(name=c.name))
def add_code_list_item(self, elem):
'''Adds ALL available codelist items from the file IF element's name
matches model name in our database (RegionVocabulary, Country, etc.)
'''
tag = elem.tag
item = None
code = smart_text(self.return_first(elem.xpath('code/text()')))
name = smart_text(self.return_first(elem.xpath('name/text()')))
description = smart_text(
self.return_first(elem.xpath('description/text()'))) or ''
language_name = smart_text(
self.return_first(elem.xpath('language/text()')))
category = smart_text(self.return_first(elem.xpath('category/text()')))
url = smart_text(self.return_first(elem.xpath('url/text()'))) or ' '
model_name = tag
# If Codelist name in IATI's file is different from our model name (or
# if other modifications are needed), such conditions should be
# overriden here:
if tag == "Country":
name = name.lower().title()
item = Country(language=language_name, data_source="IATI")
elif tag == "DocumentCategory-category":
model_name = 'DocumentCategoryCategory'
elif tag == "FileFormat":
FileFormat(category=category)
category = None
elif tag == "OrganisationRegistrationAgency":
OrganisationRegistrationAgency(category=category)
elif tag == "LocationType-category":
model_name = 'LocationTypeCategory'
elif tag == "OrganisationIdentifier":
item = OrganisationIdentifier(abbreviation=None)
elif tag == "IATIOrganisationIdentifier":
model_name = 'OrganisationIdentifier'
item = OrganisationIdentifier(abbreviation=None)
elif tag == "SectorCategory":
name = name.lower().capitalize()
elif tag == "BudgetIdentifierSector-category":
model_name = 'BudgetIdentifierSectorCategory'
elif tag == "FinanceType-category":
model_name = 'FinanceTypeCategory'
elif tag == "FinanceType":
model_name = 'FinanceType'
elif tag == "Region":
region_voc = RegionVocabulary.objects.get(code=1)
item = Region(region_vocabulary=region_voc)
elif tag == "Sector":
sector_vocabulary = SectorVocabulary.objects.get(code=1)
item = Sector(vocabulary=sector_vocabulary)
elif tag == "AidType-category":
model_name = 'AidTypeCategory'
elif tag == "AidType":
# Currently, only officially supported vocabularies (accessable via
# AITI API) are supported:
item = AidType(vocabulary=AidTypeVocabulary.objects.get(code=1))
elif tag == "CRSAddOtherFlags":
model_name = 'OtherFlags'
elif tag == "CRSChannelCode":
name = name[:255]
elif tag == "Version":
if url is None:
url = 'http://reference.iatistandard.org/' + \
self.looping_through_version.replace('.', '')
if name is None or name == '':
logger.log(0, 'name is null in ' + tag)
name = code
model = None
try:
# to do; change app_label to iati_codelist after codelist app
# change
model = apps.get_model(app_label='iati_codelists',
model_name=model_name)
except LookupError:
pass
try:
# to do; change app_label to iati_codelist after codelist app
# change
model = apps.get_model(app_label='iati_vocabulary',
model_name=model_name)
except LookupError:
pass
if not model:
try:
model = apps.get_model(app_label='geodata',
model_name=model_name)
except LookupError:
print(''.join(['Model not found: ', model_name]))
return False
if not item:
item = model()
item.code = code
item.name = name
if len(item.name) > 200:
item.name = item.name[0:200]
print("name of code: {} , name: {} shortened to 200".format(
item.code, item.name))
item.codelist_iati_version = self.looping_through_version
item = self.add_to_model_if_field_exists(model, item, 'description',
description)
item = self.add_to_model_if_field_exists(model, item, 'url', url)
if category and model_name == 'FinanceType':
ft_cat = FinanceTypeCategory.objects.filter(
# currently 'code' field is used as a PK for this model:
code=category, ).first()
if ft_cat:
item.category = ft_cat
# FIXME: refactor this general abstract approach with models!:
if category:
item = self.add_to_model_if_field_exists(model, item,
'category_id', category)
if item is not None and not model.objects.filter(
pk=item.code).exists():
try:
item.save()
except IntegrityError as err:
print("Error: {}".format(err))
pass
def add_code_list_item(self, elem):
tag = elem.tag
item = None
code = self.return_first(elem.xpath('code/text()'))
name = self.return_first(elem.xpath('name/text()'))
description = self.return_first(elem.xpath('description/text()')) or ''
language_name = self.return_first(elem.xpath('language/text()'))
category = self.return_first(elem.xpath('category/text()'))
url = self.return_first(elem.xpath('url/text()')) or ' '
model_name = tag
if tag == "Country":
name = name.lower().title()
item = Country(language=language_name, data_source="IATI")
elif tag == "DocumentCategory-category":
model_name = 'DocumentCategoryCategory'
elif tag == "FileFormat":
FileFormat(category=category)
category = None
elif tag == "OrganisationRegistrationAgency":
OrganisationRegistrationAgency(category=category)
elif tag == "LocationType-category":
model_name = 'LocationTypeCategory'
elif tag == "OrganisationIdentifier":
item = OrganisationIdentifier(abbreviation=None)
elif tag == "IATIOrganisationIdentifier":
model_name = 'OrganisationIdentifier'
item = OrganisationIdentifier(abbreviation=None)
elif tag == "SectorCategory":
name = name.lower().capitalize()
elif tag == "BudgetIdentifierSector-category":
model_name = 'BudgetIdentifierSectorCategory'
elif tag == "FinanceType-category":
model_name = 'FinanceTypeCategory'
elif tag == "Region":
region_voc = RegionVocabulary.objects.get(code=1)
item = Region(region_vocabulary=region_voc)
elif tag == "Sector":
sector_vocabulary = SectorVocabulary.objects.get(code=1)
item = Sector(vocabulary=sector_vocabulary)
elif tag == "AidType-category":
model_name = 'AidTypeCategory'
elif tag == "CRSAddOtherFlags":
model_name = 'OtherFlags'
elif tag == "CRSChannelCode":
name = name[:255]
elif tag == "Version":
if url is None:
url = 'http://iatistandard.org/' + self.looping_through_version.replace(
'.', '')
if name is None or name == '':
logger.log(0, 'name is null in ' + tag)
name = code
model = None
try:
# to do; change app_label to iati_codelist after codelist app change
model = apps.get_model(app_label='iati_codelists',
model_name=model_name)
except LookupError:
pass
try:
# to do; change app_label to iati_codelist after codelist app change
model = apps.get_model(app_label='iati_vocabulary',
model_name=model_name)
except LookupError:
pass
if not model:
try:
model = apps.get_model(app_label='geodata',
model_name=model_name)
except LookupError:
print ''.join(['Model not found: ', model_name])
return False
if not item:
item = model()
item.code = code
item.name = name
item.codelist_iati_version = self.looping_through_version
item = self.add_to_model_if_field_exists(model, item, 'description',
description)
item = self.add_to_model_if_field_exists(model, item, 'url', url)
if category:
item = self.add_to_model_if_field_exists(model, item,
'category_id', category)
if item is not None and not model.objects.filter(
pk=item.code).exists():
try:
item.save()
except IntegrityError as err:
print("Error: {}".format(err))
pass
def add_code_list_item(elem):
tag = elem.tag
db_row = None
code = elem.xpath('code/text()')
name = elem.xpath('name/text()')
description = elem.xpath('description/text()')
language_name = elem.xpath('language/text()')
category = elem.xpath('category/text()')
category_name = elem.xpath('category-name/text()')
category_description = elem.xpath('category-description/text()')
abbreviation = elem.xpath('abbreviation/text()')
sector = elem.xpath('sector/text()')
url = elem.xpath('url/text()')
if not code:
code = ""
else:
code = return_first_exist(code)
if not name:
name = ""
else:
name = return_first_exist(name)
if not description:
description = ""
else:
description = return_first_exist(description)
if not language_name:
language_name = ""
else:
language_name = return_first_exist(language_name)
if not category:
category = ""
else:
category = return_first_exist(category)
if not category_name:
category_name = ""
else:
category_name = return_first_exist(category_name)
if not category_description:
category_description = ""
else:
category_description = return_first_exist(category_description)
if not abbreviation:
abbreviation = ""
else:
abbreviation = return_first_exist(abbreviation)
if not sector:
sector = ""
else:
sector = return_first_exist(sector)
try:
if tag == "ActivityDateType":
db_row = ActivityDateType(code=code, name=name)
elif tag == "ActivityStatus":
db_row = ActivityStatus(code=code,
name=name,
language=language_name)
elif tag == "Country":
name = name.lower().capitalize()
db_row = Country(code=code,
name=name,
language=language_name,
data_source="IATI")
elif tag == "BudgetType":
db_row = BudgetType(code=code,
name=name,
language=language_name)
elif tag == "CollaborationType":
db_row = CollaborationType(code=code,
name=name,
description=description,
language=language_name)
elif tag == "ConditionType":
db_row = ConditionType(code=code,
name=name,
language=language_name)
elif tag == "Currency":
db_row = Currency(code=code,
name=name,
language=language_name)
elif tag == "DescriptionType":
db_row = DescriptionType(code=code,
name=name,
description=description)
elif tag == "DisbursementChannel":
db_row = DisbursementChannel(code=code, name=name)
elif tag == "DocumentCategory-category":
db_row = DocumentCategoryCategory(code=code, name=name)
elif tag == "DocumentCategory":
dcc = DocumentCategoryCategory.objects.get(code=category)
db_row = DocumentCategory(code=code,
name=name,
description=description,
category=dcc)
elif tag == "GeographicLocationClass":
db_row = GeographicLocationClass(code=code, name=name)
elif tag == "FileFormat":
db_row = FileFormat(code=code, name=name)
elif tag == "FlowType":
db_row = FlowType(code=code,
name=name,
description=description)
elif tag == "GazetteerAgency":
db_row = GazetteerAgency(code=code, name=name)
elif tag == "GeographicalPrecision":
db_row = GeographicalPrecision(code=code,
name=name,
description=description)
elif tag == "IndicatorMeasure":
db_row = ResultIndicatorMeasure(code=code, name=name)
elif tag == "Language":
db_row = Language(code=code, name=name)
elif tag == "LocationType-category":
db_row = LocationTypeCategory(code=code, name=name)
elif tag == "LocationType":
ltc = LocationTypeCategory.objects.get(code=category)
db_row = LocationType(code=code,
name=name,
description=description,
category=ltc)
elif tag == "OrganisationIdentifier":
db_row = OrganisationIdentifier(code=code,
abbreviation=abbreviation,
name=name)
elif tag == "OrganisationRole":
db_row = OrganisationRole(code=code,
name=name,
description=description)
elif tag == "OrganisationType":
db_row = OrganisationType(code=code, name=name)
elif tag == "PolicyMarker":
db_row = PolicyMarker(code=code, name=name)
elif tag == "PolicySignificance":
db_row = PolicySignificance(code=code,
name=name,
description=description)
elif tag == "PublisherType":
db_row = PublisherType(code=code, name=name)
elif tag == "RelatedActivityType":
db_row = RelatedActivityType(code=code,
name=name,
description=description)
elif tag == "ResultType":
db_row = ResultType(code=code, name=name)
elif tag == "SectorCategory":
name = name.lower().capitalize()
db_row = SectorCategory(code=code,
name=name,
description=description)
elif tag == "TiedStatus":
db_row = TiedStatus(code=code,
name=name,
description=description)
elif tag == "TransactionType":
db_row = TransactionType(code=code,
name=name,
description=description)
elif tag == "ValueType":
db_row = ValueType(code=code,
name=name,
description=description)
elif tag == "VerificationStatus":
db_row = VerificationStatus(code=code, name=name)
elif tag == "Vocabulary":
db_row = Vocabulary(code=code, name=name)
elif tag == "ActivityScope":
db_row = ActivityScope(code=code, name=name)
elif tag == "AidTypeFlag":
db_row = AidTypeFlag(code=code, name=name)
elif tag == "BudgetIdentifier":
db_row = BudgetIdentifier(code=code,
name=name,
category=category,
sector=sector)
elif tag == "BudgetIdentifierSector-category":
db_row = BudgetIdentifierSectorCategory(code=code,
name=name)
elif tag == "BudgetIdentifierSector":
bisc = BudgetIdentifierSectorCategory.objects.get(
code=category)
db_row = BudgetIdentifierSector(code=code,
name=name,
category=bisc)
elif tag == "BudgetIdentifierVocabulary":
db_row = BudgetIdentifierVocabulary(code=code, name=name)
elif tag == "ContactType":
db_row = ContactType(code=code, name=name)
elif tag == "LoanRepaymentPeriod":
db_row = LoanRepaymentPeriod(code=code, name=name)
elif tag == "LoanRepaymentType":
db_row = LoanRepaymentType(code=code, name=name)
elif tag == "RegionVocabulary":
db_row = RegionVocabulary(code=code, name=name)
elif tag == "FinanceType":
ftc = FinanceTypeCategory.objects.get(code=category)
db_row = FinanceType(code=code, name=name, category=ftc)
elif tag == "FinanceType-category":
db_row = FinanceTypeCategory(code=code,
name=name,
description=description)
elif tag == "Region":
region_voc = RegionVocabulary.objects.get(code=1)
db_row = Region(code=code,
name=name,
region_vocabulary=region_voc)
elif tag == "AidType-category":
db_row = AidTypeCategory(code=code,
name=name,
description=description)
elif tag == "AidType":
atc = AidTypeCategory.objects.get(code=category)
db_row = AidType(code=code,
name=name,
description=description,
category=atc)
elif tag == "Sector":
sector_cat = SectorCategory.objects.get(code=category)
db_row = Sector(code=code,
name=name,
description=description,
category=sector_cat)
# v1.04 added codelists
elif tag == "GeographicLocationReach":
db_row = GeographicLocationReach(code=code, name=name)
elif tag == "OrganisationRegistrationAgency":
db_row = OrganisationRegistrationAgency(
code=code,
name=name,
description=description,
category=category,
category_name=category_name,
url=url)
elif tag == "GeographicExactness":
db_row = GeographicExactness(code=code,
name=name,
description=description,
category=category,
url=url)
elif tag == "GeographicVocabulary":
db_row = GeographicVocabulary(code=code,
name=name,
description=description,
category=category,
url=url)
else:
print "type not saved: " + tag
if (db_row is not None):
db_row.save()
except Exception as e:
logger.info("error in codelists")
logger.info('%s (%s)' % (e.message, type(e)))
def add_missing_items():
if not Country.objects.filter(code="XK").exists():
kosovo = Country(code="XK", name="Kosovo", language="en")
kosovo.save()
def add_missing_items():
if not Country.objects.filter(code="XK").exists():
kosovo = Country(code="XK", name="Kosovo", language="en")
kosovo.save()
def add_code_list_item(self, elem):
'''Adds ALL available codelist items from the file IF element's name
matches model name in our database (RegionVocabulary, Country, etc.)
'''
tag = elem.tag
item = None
code = smart_text(self.return_first(elem.xpath('code/text()')))
name = smart_text(self.return_first(elem.xpath('name/text()')))
description = smart_text(self.return_first(
elem.xpath('description/text()'))) or ''
language_name = smart_text(
self.return_first(elem.xpath('language/text()')))
category = smart_text(self.return_first(elem.xpath('category/text()')))
url = smart_text(self.return_first(elem.xpath('url/text()'))) or ' '
model_name = tag
# If Codelist name in IATI's file is different from our model name (or
# if other modifications are needed), such conditions should be
# overriden here:
if tag == "Country":
name = name.lower().title()
item = Country(language=language_name, data_source="IATI")
elif tag == "DocumentCategory-category":
model_name = 'DocumentCategoryCategory'
elif tag == "FileFormat":
FileFormat(category=category)
category = None
elif tag == "OrganisationRegistrationAgency":
OrganisationRegistrationAgency(category=category)
elif tag == "LocationType-category":
model_name = 'LocationTypeCategory'
elif tag == "OrganisationIdentifier":
item = OrganisationIdentifier(abbreviation=None)
elif tag == "IATIOrganisationIdentifier":
model_name = 'OrganisationIdentifier'
item = OrganisationIdentifier(abbreviation=None)
elif tag == "SectorCategory":
name = name.lower().capitalize()
elif tag == "BudgetIdentifierSector-category":
model_name = 'BudgetIdentifierSectorCategory'
elif tag == "FinanceType-category":
model_name = 'FinanceTypeCategory'
elif tag == "FinanceType":
model_name = 'FinanceType'
elif tag == "Region":
region_voc = RegionVocabulary.objects.get(code=1)
item = Region(region_vocabulary=region_voc)
elif tag == "Sector":
sector_vocabulary = SectorVocabulary.objects.get(code=1)
item = Sector(vocabulary=sector_vocabulary)
elif tag == "AidType-category":
model_name = 'AidTypeCategory'
elif tag == "AidType":
# Currently, only officially supported vocabularies (accessable via
# AITI API) are supported:
item = AidType(vocabulary=AidTypeVocabulary.objects.get(code=1))
elif tag == "CRSAddOtherFlags":
model_name = 'OtherFlags'
elif tag == "CRSChannelCode":
name = name[:255]
elif tag == "Version":
if url is None:
url = 'http://reference.iatistandard.org/' + \
self.looping_through_version.replace('.', '')
if name is None or name == '':
logger.log(0, 'name is null in ' + tag)
name = code
model = None
try:
# to do; change app_label to iati_codelist after codelist app
# change
model = apps.get_model(
app_label='iati_codelists', model_name=model_name)
except LookupError:
pass
try:
# to do; change app_label to iati_codelist after codelist app
# change
model = apps.get_model(
app_label='iati_vocabulary', model_name=model_name)
except LookupError:
pass
if not model:
try:
model = apps.get_model(
app_label='geodata', model_name=model_name)
except LookupError:
print(''.join(['Model not found: ', model_name]))
return False
if not item:
item = model()
item.code = code
item.name = name
if len(item.name) > 200:
item.name = item.name[0:200]
print("name of code: {} , name: {} shortened to 200".format(
item.code, item.name))
item.codelist_iati_version = self.looping_through_version
item = self.add_to_model_if_field_exists(
model, item, 'description', description)
item = self.add_to_model_if_field_exists(model, item, 'url', url)
if category and model_name == 'FinanceType':
ft_cat = FinanceTypeCategory.objects.filter(
# currently 'code' field is used as a PK for this model:
code=category,
).first()
if ft_cat:
item.category = ft_cat
# FIXME: refactor this general abstract approach with models!:
if category:
item = self.add_to_model_if_field_exists(
model, item, 'category_id', category)
if item is not None and not model.objects.filter(
pk=item.code
).exists():
try:
item.save()
except IntegrityError as err:
print("Error: {}".format(err))
pass