def add_code_list_item(self, elem): tag = elem.tag item = None code = self.return_first(elem.xpath('code/text()')) name = self.return_first(elem.xpath('name/text()')) description = self.return_first(elem.xpath('description/text()')) or '' language_name = self.return_first(elem.xpath('language/text()')) category = self.return_first(elem.xpath('category/text()')) url = self.return_first(elem.xpath('url/text()')) or ' ' model_name = tag if tag == "Country": name = name.lower().capitalize() item = Country(language=language_name, data_source="IATI") elif tag == "Region": region_voc = RegionVocabulary.objects.get(code=1) item = Region(region_vocabulary=region_voc) if name is None or name == '': logger.log(0, 'name is null in ' + tag) name = code model = None try: # to do; change app_label to iati_codelist after codelist app change model = apps.get_model(app_label='geodata', model_name=model_name) except LookupError: pass if not model: try: model = apps.get_model(app_label='geodata', model_name=model_name) except LookupError: print(''.join(['Model not found: ', model_name])) return False if not item: item = model() item.code = code item.name = name item.codelist_iati_version = self.looping_through_version item = self.add_to_model_if_field_exists(model, item, 'description', description) item = self.add_to_model_if_field_exists(model, item, 'url', url) if category: item = self.add_to_model_if_field_exists(model, item, 'category_id', category) if item is not None and not model.objects.filter( pk=item.code).exists(): try: item.save() except IntegrityError as err: print("Error: {}".format(err)) pass
def update_polygon(self): admin_countries = self.get_json_data( "/../data_backup/allcountrycodes.json") for k in admin_countries: # .get('features'): country_iso2 = k.get('alpha-2').lower() name = k.get('name').lower() country_iso3 = k.get('alpha-3').lower() if not country_iso2: continue c, created = Country.objects.get_or_create(iso2=country_iso2, iso3=country_iso3, name=name, primary_name=True) if created: c.save() # Add or update geolocation try: geolocation = Geolocation.objects.get(tag=c.name) except Geolocation.DoesNotExist: geolocation = Geolocation(content_object=c, tag=c.name, type='country') geolocation.save() print('Country : {name}:'.format(name=c.name)) poly_countries = self.get_json_data( "/../data_backup/country_data.json").get('features') for k in poly_countries: # .get('features'): if 'iso2' in k.get('properties'): iso2 = k.get('properties').get('iso2').lower() filtered_set = Country.objects.filter(iso2=iso2, primary_name=True) if len(filtered_set) > 0: c = filtered_set[0] else: name = k.get('properties').get('name').lower() c = Country(name=name, iso2=iso2, primary_name=True) c.save() c.polygons = json.dumps(k.get('geometry')) c.save() # Add or update geolocation try: geolocation = Geolocation.objects.get(tag=c.name) except Geolocation.DoesNotExist: geolocation = Geolocation(content_object=c, tag=c.name, type='country') geolocation.save() print('Polygon : {name}:'.format(name=c.name))
def add_code_list_item(self, elem): '''Adds ALL available codelist items from the file IF element's name matches model name in our database (RegionVocabulary, Country, etc.) ''' tag = elem.tag item = None code = smart_text(self.return_first(elem.xpath('code/text()'))) name = smart_text(self.return_first(elem.xpath('name/text()'))) description = smart_text( self.return_first(elem.xpath('description/text()'))) or '' language_name = smart_text( self.return_first(elem.xpath('language/text()'))) category = smart_text(self.return_first(elem.xpath('category/text()'))) url = smart_text(self.return_first(elem.xpath('url/text()'))) or ' ' model_name = tag # If Codelist name in IATI's file is different from our model name (or # if other modifications are needed), such conditions should be # overriden here: if tag == "Country": name = name.lower().title() item = Country(language=language_name, data_source="IATI") elif tag == "DocumentCategory-category": model_name = 'DocumentCategoryCategory' elif tag == "FileFormat": FileFormat(category=category) category = None elif tag == "OrganisationRegistrationAgency": OrganisationRegistrationAgency(category=category) elif tag == "LocationType-category": model_name = 'LocationTypeCategory' elif tag == "OrganisationIdentifier": item = OrganisationIdentifier(abbreviation=None) elif tag == "IATIOrganisationIdentifier": model_name = 'OrganisationIdentifier' item = OrganisationIdentifier(abbreviation=None) elif tag == "SectorCategory": name = name.lower().capitalize() elif tag == "BudgetIdentifierSector-category": model_name = 'BudgetIdentifierSectorCategory' elif tag == "FinanceType-category": model_name = 'FinanceTypeCategory' elif tag == "FinanceType": model_name = 'FinanceType' elif tag == "Region": region_voc = RegionVocabulary.objects.get(code=1) item = Region(region_vocabulary=region_voc) elif tag == "Sector": sector_vocabulary = SectorVocabulary.objects.get(code=1) item = Sector(vocabulary=sector_vocabulary) elif tag == "AidType-category": model_name = 'AidTypeCategory' elif tag == "AidType": # Currently, only officially supported vocabularies (accessable via # AITI API) are supported: item = AidType(vocabulary=AidTypeVocabulary.objects.get(code=1)) elif tag == "CRSAddOtherFlags": model_name = 'OtherFlags' elif tag == "CRSChannelCode": name = name[:255] elif tag == "Version": if url is None: url = 'http://reference.iatistandard.org/' + \ self.looping_through_version.replace('.', '') if name is None or name == '': logger.log(0, 'name is null in ' + tag) name = code model = None try: # to do; change app_label to iati_codelist after codelist app # change model = apps.get_model(app_label='iati_codelists', model_name=model_name) except LookupError: pass try: # to do; change app_label to iati_codelist after codelist app # change model = apps.get_model(app_label='iati_vocabulary', model_name=model_name) except LookupError: pass if not model: try: model = apps.get_model(app_label='geodata', model_name=model_name) except LookupError: print(''.join(['Model not found: ', model_name])) return False if not item: item = model() item.code = code item.name = name if len(item.name) > 200: item.name = item.name[0:200] print("name of code: {} , name: {} shortened to 200".format( item.code, item.name)) item.codelist_iati_version = self.looping_through_version item = self.add_to_model_if_field_exists(model, item, 'description', description) item = self.add_to_model_if_field_exists(model, item, 'url', url) if category and model_name == 'FinanceType': ft_cat = FinanceTypeCategory.objects.filter( # currently 'code' field is used as a PK for this model: code=category, ).first() if ft_cat: item.category = ft_cat # FIXME: refactor this general abstract approach with models!: if category: item = self.add_to_model_if_field_exists(model, item, 'category_id', category) if item is not None and not model.objects.filter( pk=item.code).exists(): try: item.save() except IntegrityError as err: print("Error: {}".format(err)) pass
def add_code_list_item(self, elem): tag = elem.tag item = None code = self.return_first(elem.xpath('code/text()')) name = self.return_first(elem.xpath('name/text()')) description = self.return_first(elem.xpath('description/text()')) or '' language_name = self.return_first(elem.xpath('language/text()')) category = self.return_first(elem.xpath('category/text()')) url = self.return_first(elem.xpath('url/text()')) or ' ' model_name = tag if tag == "Country": name = name.lower().title() item = Country(language=language_name, data_source="IATI") elif tag == "DocumentCategory-category": model_name = 'DocumentCategoryCategory' elif tag == "FileFormat": FileFormat(category=category) category = None elif tag == "OrganisationRegistrationAgency": OrganisationRegistrationAgency(category=category) elif tag == "LocationType-category": model_name = 'LocationTypeCategory' elif tag == "OrganisationIdentifier": item = OrganisationIdentifier(abbreviation=None) elif tag == "IATIOrganisationIdentifier": model_name = 'OrganisationIdentifier' item = OrganisationIdentifier(abbreviation=None) elif tag == "SectorCategory": name = name.lower().capitalize() elif tag == "BudgetIdentifierSector-category": model_name = 'BudgetIdentifierSectorCategory' elif tag == "FinanceType-category": model_name = 'FinanceTypeCategory' elif tag == "Region": region_voc = RegionVocabulary.objects.get(code=1) item = Region(region_vocabulary=region_voc) elif tag == "Sector": sector_vocabulary = SectorVocabulary.objects.get(code=1) item = Sector(vocabulary=sector_vocabulary) elif tag == "AidType-category": model_name = 'AidTypeCategory' elif tag == "CRSAddOtherFlags": model_name = 'OtherFlags' elif tag == "CRSChannelCode": name = name[:255] elif tag == "Version": if url is None: url = 'http://iatistandard.org/' + self.looping_through_version.replace( '.', '') if name is None or name == '': logger.log(0, 'name is null in ' + tag) name = code model = None try: # to do; change app_label to iati_codelist after codelist app change model = apps.get_model(app_label='iati_codelists', model_name=model_name) except LookupError: pass try: # to do; change app_label to iati_codelist after codelist app change model = apps.get_model(app_label='iati_vocabulary', model_name=model_name) except LookupError: pass if not model: try: model = apps.get_model(app_label='geodata', model_name=model_name) except LookupError: print ''.join(['Model not found: ', model_name]) return False if not item: item = model() item.code = code item.name = name item.codelist_iati_version = self.looping_through_version item = self.add_to_model_if_field_exists(model, item, 'description', description) item = self.add_to_model_if_field_exists(model, item, 'url', url) if category: item = self.add_to_model_if_field_exists(model, item, 'category_id', category) if item is not None and not model.objects.filter( pk=item.code).exists(): try: item.save() except IntegrityError as err: print("Error: {}".format(err)) pass
def add_code_list_item(elem): tag = elem.tag db_row = None code = elem.xpath('code/text()') name = elem.xpath('name/text()') description = elem.xpath('description/text()') language_name = elem.xpath('language/text()') category = elem.xpath('category/text()') category_name = elem.xpath('category-name/text()') category_description = elem.xpath('category-description/text()') abbreviation = elem.xpath('abbreviation/text()') sector = elem.xpath('sector/text()') url = elem.xpath('url/text()') if not code: code = "" else: code = return_first_exist(code) if not name: name = "" else: name = return_first_exist(name) if not description: description = "" else: description = return_first_exist(description) if not language_name: language_name = "" else: language_name = return_first_exist(language_name) if not category: category = "" else: category = return_first_exist(category) if not category_name: category_name = "" else: category_name = return_first_exist(category_name) if not category_description: category_description = "" else: category_description = return_first_exist(category_description) if not abbreviation: abbreviation = "" else: abbreviation = return_first_exist(abbreviation) if not sector: sector = "" else: sector = return_first_exist(sector) try: if tag == "ActivityDateType": db_row = ActivityDateType(code=code, name=name) elif tag == "ActivityStatus": db_row = ActivityStatus(code=code, name=name, language=language_name) elif tag == "Country": name = name.lower().capitalize() db_row = Country(code=code, name=name, language=language_name, data_source="IATI") elif tag == "BudgetType": db_row = BudgetType(code=code, name=name, language=language_name) elif tag == "CollaborationType": db_row = CollaborationType(code=code, name=name, description=description, language=language_name) elif tag == "ConditionType": db_row = ConditionType(code=code, name=name, language=language_name) elif tag == "Currency": db_row = Currency(code=code, name=name, language=language_name) elif tag == "DescriptionType": db_row = DescriptionType(code=code, name=name, description=description) elif tag == "DisbursementChannel": db_row = DisbursementChannel(code=code, name=name) elif tag == "DocumentCategory-category": db_row = DocumentCategoryCategory(code=code, name=name) elif tag == "DocumentCategory": dcc = DocumentCategoryCategory.objects.get(code=category) db_row = DocumentCategory(code=code, name=name, description=description, category=dcc) elif tag == "GeographicLocationClass": db_row = GeographicLocationClass(code=code, name=name) elif tag == "FileFormat": db_row = FileFormat(code=code, name=name) elif tag == "FlowType": db_row = FlowType(code=code, name=name, description=description) elif tag == "GazetteerAgency": db_row = GazetteerAgency(code=code, name=name) elif tag == "GeographicalPrecision": db_row = GeographicalPrecision(code=code, name=name, description=description) elif tag == "IndicatorMeasure": db_row = ResultIndicatorMeasure(code=code, name=name) elif tag == "Language": db_row = Language(code=code, name=name) elif tag == "LocationType-category": db_row = LocationTypeCategory(code=code, name=name) elif tag == "LocationType": ltc = LocationTypeCategory.objects.get(code=category) db_row = LocationType(code=code, name=name, description=description, category=ltc) elif tag == "OrganisationIdentifier": db_row = OrganisationIdentifier(code=code, abbreviation=abbreviation, name=name) elif tag == "OrganisationRole": db_row = OrganisationRole(code=code, name=name, description=description) elif tag == "OrganisationType": db_row = OrganisationType(code=code, name=name) elif tag == "PolicyMarker": db_row = PolicyMarker(code=code, name=name) elif tag == "PolicySignificance": db_row = PolicySignificance(code=code, name=name, description=description) elif tag == "PublisherType": db_row = PublisherType(code=code, name=name) elif tag == "RelatedActivityType": db_row = RelatedActivityType(code=code, name=name, description=description) elif tag == "ResultType": db_row = ResultType(code=code, name=name) elif tag == "SectorCategory": name = name.lower().capitalize() db_row = SectorCategory(code=code, name=name, description=description) elif tag == "TiedStatus": db_row = TiedStatus(code=code, name=name, description=description) elif tag == "TransactionType": db_row = TransactionType(code=code, name=name, description=description) elif tag == "ValueType": db_row = ValueType(code=code, name=name, description=description) elif tag == "VerificationStatus": db_row = VerificationStatus(code=code, name=name) elif tag == "Vocabulary": db_row = Vocabulary(code=code, name=name) elif tag == "ActivityScope": db_row = ActivityScope(code=code, name=name) elif tag == "AidTypeFlag": db_row = AidTypeFlag(code=code, name=name) elif tag == "BudgetIdentifier": db_row = BudgetIdentifier(code=code, name=name, category=category, sector=sector) elif tag == "BudgetIdentifierSector-category": db_row = BudgetIdentifierSectorCategory(code=code, name=name) elif tag == "BudgetIdentifierSector": bisc = BudgetIdentifierSectorCategory.objects.get( code=category) db_row = BudgetIdentifierSector(code=code, name=name, category=bisc) elif tag == "BudgetIdentifierVocabulary": db_row = BudgetIdentifierVocabulary(code=code, name=name) elif tag == "ContactType": db_row = ContactType(code=code, name=name) elif tag == "LoanRepaymentPeriod": db_row = LoanRepaymentPeriod(code=code, name=name) elif tag == "LoanRepaymentType": db_row = LoanRepaymentType(code=code, name=name) elif tag == "RegionVocabulary": db_row = RegionVocabulary(code=code, name=name) elif tag == "FinanceType": ftc = FinanceTypeCategory.objects.get(code=category) db_row = FinanceType(code=code, name=name, category=ftc) elif tag == "FinanceType-category": db_row = FinanceTypeCategory(code=code, name=name, description=description) elif tag == "Region": region_voc = RegionVocabulary.objects.get(code=1) db_row = Region(code=code, name=name, region_vocabulary=region_voc) elif tag == "AidType-category": db_row = AidTypeCategory(code=code, name=name, description=description) elif tag == "AidType": atc = AidTypeCategory.objects.get(code=category) db_row = AidType(code=code, name=name, description=description, category=atc) elif tag == "Sector": sector_cat = SectorCategory.objects.get(code=category) db_row = Sector(code=code, name=name, description=description, category=sector_cat) # v1.04 added codelists elif tag == "GeographicLocationReach": db_row = GeographicLocationReach(code=code, name=name) elif tag == "OrganisationRegistrationAgency": db_row = OrganisationRegistrationAgency( code=code, name=name, description=description, category=category, category_name=category_name, url=url) elif tag == "GeographicExactness": db_row = GeographicExactness(code=code, name=name, description=description, category=category, url=url) elif tag == "GeographicVocabulary": db_row = GeographicVocabulary(code=code, name=name, description=description, category=category, url=url) else: print "type not saved: " + tag if (db_row is not None): db_row.save() except Exception as e: logger.info("error in codelists") logger.info('%s (%s)' % (e.message, type(e)))
def add_missing_items(): if not Country.objects.filter(code="XK").exists(): kosovo = Country(code="XK", name="Kosovo", language="en") kosovo.save()
def add_code_list_item(self, elem): '''Adds ALL available codelist items from the file IF element's name matches model name in our database (RegionVocabulary, Country, etc.) ''' tag = elem.tag item = None code = smart_text(self.return_first(elem.xpath('code/text()'))) name = smart_text(self.return_first(elem.xpath('name/text()'))) description = smart_text(self.return_first( elem.xpath('description/text()'))) or '' language_name = smart_text( self.return_first(elem.xpath('language/text()'))) category = smart_text(self.return_first(elem.xpath('category/text()'))) url = smart_text(self.return_first(elem.xpath('url/text()'))) or ' ' model_name = tag # If Codelist name in IATI's file is different from our model name (or # if other modifications are needed), such conditions should be # overriden here: if tag == "Country": name = name.lower().title() item = Country(language=language_name, data_source="IATI") elif tag == "DocumentCategory-category": model_name = 'DocumentCategoryCategory' elif tag == "FileFormat": FileFormat(category=category) category = None elif tag == "OrganisationRegistrationAgency": OrganisationRegistrationAgency(category=category) elif tag == "LocationType-category": model_name = 'LocationTypeCategory' elif tag == "OrganisationIdentifier": item = OrganisationIdentifier(abbreviation=None) elif tag == "IATIOrganisationIdentifier": model_name = 'OrganisationIdentifier' item = OrganisationIdentifier(abbreviation=None) elif tag == "SectorCategory": name = name.lower().capitalize() elif tag == "BudgetIdentifierSector-category": model_name = 'BudgetIdentifierSectorCategory' elif tag == "FinanceType-category": model_name = 'FinanceTypeCategory' elif tag == "FinanceType": model_name = 'FinanceType' elif tag == "Region": region_voc = RegionVocabulary.objects.get(code=1) item = Region(region_vocabulary=region_voc) elif tag == "Sector": sector_vocabulary = SectorVocabulary.objects.get(code=1) item = Sector(vocabulary=sector_vocabulary) elif tag == "AidType-category": model_name = 'AidTypeCategory' elif tag == "AidType": # Currently, only officially supported vocabularies (accessable via # AITI API) are supported: item = AidType(vocabulary=AidTypeVocabulary.objects.get(code=1)) elif tag == "CRSAddOtherFlags": model_name = 'OtherFlags' elif tag == "CRSChannelCode": name = name[:255] elif tag == "Version": if url is None: url = 'http://reference.iatistandard.org/' + \ self.looping_through_version.replace('.', '') if name is None or name == '': logger.log(0, 'name is null in ' + tag) name = code model = None try: # to do; change app_label to iati_codelist after codelist app # change model = apps.get_model( app_label='iati_codelists', model_name=model_name) except LookupError: pass try: # to do; change app_label to iati_codelist after codelist app # change model = apps.get_model( app_label='iati_vocabulary', model_name=model_name) except LookupError: pass if not model: try: model = apps.get_model( app_label='geodata', model_name=model_name) except LookupError: print(''.join(['Model not found: ', model_name])) return False if not item: item = model() item.code = code item.name = name if len(item.name) > 200: item.name = item.name[0:200] print("name of code: {} , name: {} shortened to 200".format( item.code, item.name)) item.codelist_iati_version = self.looping_through_version item = self.add_to_model_if_field_exists( model, item, 'description', description) item = self.add_to_model_if_field_exists(model, item, 'url', url) if category and model_name == 'FinanceType': ft_cat = FinanceTypeCategory.objects.filter( # currently 'code' field is used as a PK for this model: code=category, ).first() if ft_cat: item.category = ft_cat # FIXME: refactor this general abstract approach with models!: if category: item = self.add_to_model_if_field_exists( model, item, 'category_id', category) if item is not None and not model.objects.filter( pk=item.code ).exists(): try: item.save() except IntegrityError as err: print("Error: {}".format(err)) pass