def fillup_alt_name(self): """ Make sure that for every language there is a least ONE entry in the altnames table for each geoitem. Currently, when there is no translation, any entry in altname is often missing (for example there is no entry LIKE "%rotenburg%" in altname altogether. Loop all three (country, region, city) tables and loop all optional languages for the installation, and then check that there is an entry for the geoitem and language in altname and if not, then add the default (english) name from the (country, region, city) table. The reason is to be able to select just from altname for autocomplete and never query the names from (country, region, city). """ languages = [e[0] for e in settings.LANGUAGES] types = ((1, 'country'), (2, 'region'), (3, 'city')) alt = AltName.objects.all() obj = { 'country': Country.objects.all(), 'region': Region.objects.all(), 'city': City.objects.all(), } for t in types: for c in obj[t[1]]: for lg in languages: print('[t={0}] [c={1}] [lg={2}] Check entry for "{3}"...'. format(t[1], c.id, lg, c.name)) for e in alt: if e.geoname_id == c.id and e.language == lg: print( 'Entry found with AltName id [{0}] as "{1}".'. format(e.id, e.name)) break else: # No entries, add one. print('NO entry found, so add one...') addalt = AltName() addalt.geoname_id = c.id addalt.language = lg addalt.crc = '' addalt.name = c.name addalt.slug = slugify(c.name) addalt.type = t[0] addalt.is_main = False addalt.is_preferred = True addalt.is_short = True addalt.is_colloquial = False addalt.is_historic = False if t == 2 or t == 3: # If this is a city or region, find the country addalt.country_id = c.country_id if t == 3: # If this is a city, also find the region. addalt.region_id = c.region_id addalt.save() print('New entry added for "{0}".'.format(c.name))
def fillup_alt_name(self): """ Make sure that for every language there is a least ONE entry in the altnames table for each geoitem. Currently, when there is no translation, any entry in altname is often missing (for example there is no entry LIKE "%rotenburg%" in altname altogether. Loop all three (country, region, city) tables and loop all optional languages for the installation, and then check that there is an entry for the geoitem and language in altname and if not, then add the default (english) name from the (country, region, city) table. The reason is to be able to select just from altname for autocomplete and never query the names from (country, region, city). """ languages = [e[0] for e in settings.LANGUAGES] types = ((1, 'country'), (2, 'region'), (3, 'city')) alt = AltName.objects.all() obj = {'country': Country.objects.all(), 'region': Region.objects.all(), 'city': City.objects.all(), } for t in types: for c in obj[t[1]]: for lg in languages: print('[t={0}] [c={1}] [lg={2}] Check entry for "{3}"...' .format(t[1], c.id, lg, c.name)) for e in alt: if e.geoname_id == c.id and e.language == lg: print('Entry found with AltName id [{0}] as "{1}".' .format(e.id, e.name)) break else: # No entries, add one. print('NO entry found, so add one...') addalt = AltName() addalt.geoname_id = c.id addalt.language = lg addalt.crc = '' addalt.name = c.name addalt.slug = slugify(c.name) addalt.type = t[0] addalt.is_main = False addalt.is_preferred = True addalt.is_short = True addalt.is_colloquial = False addalt.is_historic = False if t == 2 or t == 3: # If this is a city or region, find the country addalt.country_id = c.country_id if t == 3: # If this is a city, also find the region. addalt.region_id = c.region_id addalt.save() print('New entry added for "{0}".'.format(c.name))
def import_alt_name(self): i = 0 # Download the altnames file if necessary and fetch the data. uptodate = self.download('alt_name') if uptodate and not self.force: return print('Fetching fresh alt_name data...') data = self.get_data('alt_name') print('Successfully fetched alt_name data.') # Import only names in the languages set in settings.LANGUAGES languages = [e[0] for e in settings.LANGUAGES] print('Looking for languages: {0}'.format(languages)) # The geo types used. types = ((1, 'country'), (2, 'region'), (3, 'city')) # Load only geoname_id numbers from country, region, city into memory. print('Building indexes...') self.build_geo_index() self.build_country_index() self.build_region_index() print('All indexes built.') print('Start importing of AltName data.') for items in self.parse(data): i += 1 print('{} import geoname_id "{}" for language {}' .format(i, items[1], items[2]), end=" ") # Verify that the "name" items[3] contains a string: item_name = items[3].strip() if not item_name: print('SKIP: Item had an empty strng for a name.') continue # Only get names for languages in use. if items[2] not in languages: print('SKIP: Unknown language.') continue # The geoname_id of the item. item_geoname_id = int(items[1]) if not item_geoname_id: print('SKIP: Item had an empty strng for a name.') continue # Find type (city, region, or country) for the item. Must be one # of the three types defined above. item_type = None for t in types: # The geo_index contains all geoname_id vals by type. if item_geoname_id in self.geo_index[t[1]]: # Remember "types" id (city, region, coutnry) of the item. item_type = t[0] break if item_type is None: print('SKIP: Geoname type not found.') continue print('type "{}"'.format(item_type), end=' ') # All import data clean, create database object. alt = AltName() # Use altname_id from source database as pk. Not useful, because # the pk is actually never used hereafter. # alt.id = int(items[0]) # The important identifier, the geoname_id of the geo object. alt.geoname_id = item_geoname_id alt.language = items[2] alt.crc = '' alt.name = item_name # items[3] alt.slug = slugify(item_name) alt.type = item_type alt.is_main = bool(0) alt.is_preferred = bool(items[4]) alt.is_short = bool(items[5]) alt.is_colloquial = bool(items[6]) alt.is_historic = bool(items[7]) alt.save() print('SAVED!')
def import_alt_name(self): i = 0 # Download the altnames file if necessary and fetch the data. uptodate = self.download('alt_name') if uptodate and not self.force: return print('Fetching fresh alt_name data...') data = self.get_data('alt_name') print('Successfully fetched alt_name data.') # Import only names in the languages set in settings.LANGUAGES languages = [e[0] for e in settings.LANGUAGES] print('Looking for languages: {0}'.format(languages)) # The geo types used. types = ((1, 'country'), (2, 'region'), (3, 'city')) # Load only geoname_id numbers from country, region, city into memory. print('Building indexes...') self.build_geo_index() self.build_country_index() self.build_region_index() print('All indexes built.') print('Start importing of AltName data.') for items in self.parse(data): i += 1 print('{} import geoname_id "{}" for language {}'.format( i, items[1], items[2]), end=" ") # Verify that the "name" items[3] contains a string: item_name = items[3].strip() if not item_name: print('SKIP: Item had an empty strng for a name.') continue # Only get names for languages in use. if items[2] not in languages: print('SKIP: Unknown language.') continue # The geoname_id of the item. item_geoname_id = int(items[1]) if not item_geoname_id: print('SKIP: Item had an empty strng for a name.') continue # Find type (city, region, or country) for the item. Must be one # of the three types defined above. item_type = None for t in types: # The geo_index contains all geoname_id vals by type. if item_geoname_id in self.geo_index[t[1]]: # Remember "types" id (city, region, coutnry) of the item. item_type = t[0] break if item_type is None: print('SKIP: Geoname type not found.') continue print('type "{}"'.format(item_type), end=' ') # All import data clean, create database object. alt = AltName() # Use altname_id from source database as pk. Not useful, because # the pk is actually never used hereafter. # alt.id = int(items[0]) # The important identifier, the geoname_id of the geo object. alt.geoname_id = item_geoname_id alt.language = items[2] alt.crc = '' alt.name = item_name # items[3] alt.slug = slugify(item_name) alt.type = item_type alt.is_main = bool(0) alt.is_preferred = bool(items[4]) alt.is_short = bool(items[5]) alt.is_colloquial = bool(items[6]) alt.is_historic = bool(items[7]) alt.save() print('SAVED!')