class WhereaboutsFactory(DjangoModelFactory):
class Meta:
model = 'hosting.Whereabouts'
type = Faker('random_element', elements=[ch.value for ch in LocationType])
@factory.lazy_attribute
def name(self):
return Faker._get_faker().city().upper()
@factory.lazy_attribute
def state(self):
if self.country in countries_with_mandatory_region():
return CountryRegionFactory(country=self.country).iso_code
else:
return ""
country = Faker('random_element', elements=COUNTRIES.keys())
@factory.lazy_attribute
def bbox(self):
minx, miny, maxx, maxy = self.center.buffer(width=uniform_random(
0.05, 0.45),
quadsegs=2).extent
return LineString((minx, miny), (maxx, maxy), srid=SRID)
@factory.lazy_attribute
def center(self):
# Cannot use the 'local_latlng' Faker, they don't have all countries in the database!
return Point([
uniform_random(a, b)
for a, b in zip(*COUNTRIES_GEO[self.country]['bbox'].values())
],
srid=SRID)
def add_arguments(self, parser):
"""
The command can be given a list of ISO country codes to update,
as an argument. A missing argument or the word "ALL" mean that
all countries in the local database will be refreshed.
"""
parser.add_argument(
'update_countries',
nargs='*',
choices=tuple(sorted(COUNTRIES.keys())) + ('ALL', ),
default='ALL',
metavar='country code',
help=
"one or more country codes to update, omit to update all countries."
)
parser.add_argument(
'-s',
'--start-from',
default='AA',
choices='ABCDEFGHIJKLMNOPQRSTUVWXYZ',
help="when all countries are being updated, begin with the ISO code "
"starting with this letter.")
parser.add_argument(
'-e',
'--end-before',
default='ZZ',
choices='ABCDEFGHIJKLMNOPQRSTUVWXYZ',
help=
"when all countries are being updated, stop before the ISO code "
"starting with this letter.")
class TravelAdviceFactory(DjangoModelFactory):
class Meta:
model = 'hosting.TravelAdvice'
class Params:
in_past, in_present, in_future = None, None, None
content = Faker('paragraph')
description = factory.LazyAttribute(
lambda obj: '<p>{}</p>'.format(obj.content))
countries = Faker('random_elements',
elements=COUNTRIES.keys(),
unique=True,
length=factory.LazyFunction(lambda: randint(1, 4)))
@factory.lazy_attribute
def active_from(self):
faker = Faker._get_faker()
if self.in_past:
faked_date = faker.optional_value('date_between',
start_date='-365d',
end_date='-200d')
elif self.in_future:
faked_date = faker.date_between(start_date='+2d', end_date='+199d')
elif self.in_present:
faked_date = faker.optional_value('date_between',
start_date='-200d',
end_date='-2d')
else:
faked_date = faker.optional_value('date_object',
end_datetime='+5y')
return faked_date
@factory.lazy_attribute
def active_until(self):
faker = Faker._get_faker()
if self.in_past:
faked_date = faker.date_between(start_date='-199d', end_date='-2d')
elif self.in_future:
faked_date = faker.optional_value('date_between',
start_date='+200d',
end_date='+365d')
elif self.in_present:
faked_date = faker.optional_value('date_between',
start_date='+2d',
end_date='+200d')
else:
if self.active_from:
start, end = self.active_from, self.active_from + timedelta(
days=365)
faked_date = faker.optional_value('date_between_dates',
date_start=start,
date_end=end)
else:
faked_date = faker.optional_value('date_object',
end_datetime='+5y')
return faked_date
class PlaceFactory(DjangoModelFactory):
class Meta:
model = 'hosting.Place'
owner = factory.SubFactory('tests.factories.ProfileFactory')
country = Faker('random_element', elements=COUNTRIES.keys())
@factory.lazy_attribute
def state_province(self):
if self.country in countries_with_mandatory_region():
region = CountryRegionFactory(country=self.country)
return region.iso_code
else:
return ""
city = Faker('city')
address = Faker('address')
@factory.lazy_attribute
def location(self):
# Cannot use the 'local_latlng' Faker, they don't have all countries in the database!
return Point([
uniform_random(a, b)
for a, b in zip(*COUNTRIES_GEO[self.country]['bbox'].values())
],
srid=SRID)
description = Faker('paragraph', nb_sentences=4)
short_description = Faker('text', max_nb_chars=140)
in_book = False
@staticmethod
def generate_postcode(country):
regex = COUNTRIES_DATA[country]['postcode_regex'] or r'\d{5}'
# The * repetition qualifier makes the generator go wild, strictly limit to 1 copy.
regex = regex.replace('*', '{1}')
# Articially limit the length of overly permissive chunks.
regex = re.sub(r'{0,\d\d}', '{0,2}', regex)
# Generate a random value according to the constrained regular expression.
# All whitespaces are condensed to single space and the value is uppercased.
value = ""
while value in ("", "GIR0AA", "GIR 0AA"):
# The generator has a strong preference to this UK postal code...
value = ' '.join(rstr.xeger(regex).upper().strip().split())
return value
@factory.post_generation
def postcode(instance, create, value, **kwargs):
if not value:
return
if value is True:
instance.postcode = PlaceFactory.generate_postcode(
instance.country)
else:
instance.postcode = value
class CountryRegionFactory(DjangoModelFactory):
class Meta:
model = 'hosting.CountryRegion'
class Params:
short_code = factory.LazyFunction(lambda: random() < 0.20)
country = Faker('random_element', elements=COUNTRIES.keys())
iso_code = Faker('pystr_format',
string_format='???#',
letters='ABCDEFGHJKLMNPQRSTUVWXYZ')
latin_code = factory.Maybe('short_code',
yes_declaration=Faker(
'pystr_format',
string_format='??',
letters='ABCDEFGHIJKLMNOPQRSTUVWXYZ'),
no_declaration=Faker('sentence', nb_words=3))
latin_name = factory.Maybe('short_code',
yes_declaration=Faker('sentence', nb_words=3),
no_declaration="")
@factory.lazy_attribute
def esperanto_name(self):
latin_region = self.latin_name or self.latin_code
replacements = [
('Q', 'Kv'),
('q', 'kv'),
('W', 'V'),
('w', 'v'),
('X', 'Ks'),
('x', 'ks'),
('Y', 'J'),
('y ', 'i '),
('y.', 'i.'),
('y', 'j'),
('Ph', 'F'),
('ph', 'f'),
('Th', 'Z'),
('th', 'z'),
('cc', 'k'),
('ee', 'i'),
('ll', 'l'),
('tt', 't'),
(' ', '-'),
('.', 'o'),
]
for lat_letter, esp_letter in replacements:
latin_region = latin_region.replace(lat_letter, esp_letter)
return latin_region
def add_arguments(self, parser):
"""
The command can be given a list of ISO country codes to update,
as an argument. A missing argument or the word "ALL" mean that
all countries in the local database will be refreshed.
"""
parser.add_argument(
'update_countries',
nargs='*',
choices=tuple(COUNTRIES.keys()) + ('ALL', ),
default='ALL',
metavar='country code',
help=
"one or more country codes to update, omit to update all countries."
)
class PhoneFactory(DjangoModelFactory):
class Meta:
model = 'hosting.Phone'
profile = factory.SubFactory('tests.factories.ProfileFactory')
@factory.lazy_attribute
def number(self):
# the Faker's phone-number provider is a mess.
phone = PhoneNumber()
while not phone.is_valid():
phone = PhoneNumber(country_code=randint(1, 999),
national_number=randint(10000, 99999999990))
return phone
country = Faker('random_element', elements=COUNTRIES.keys())
comments = Faker('text', max_nb_chars=20)
type = Faker('random_element',
elements=[ch[0] for ch in PHONE_TYPE_CHOICES])
def construct_address_form(country_code, i18n_rules):
class_name = 'AddressForm%s' % country_code
base_class = CountryAwareAddressForm
form_kwargs = {
'Meta': type(str('Meta'), (base_class.Meta, object), {}),
'formfield_callback': None
}
class_ = type(base_class)(str(class_name), (base_class, ), form_kwargs)
update_base_fields(class_, i18n_rules)
class_.i18n_country_code = country_code
class_.i18n_fields_order = property(get_form_i18n_lines)
return class_
for country in COUNTRIES.keys():
try:
country_rules = i18naddress.get_validation_rules(
{'country_code': country})
except ValueError:
country_rules = i18naddress.get_validation_rules({})
UNKNOWN_COUNTRIES.add(country)
COUNTRY_CHOICES = [(code, label) for code, label in COUNTRIES.items()
if code not in UNKNOWN_COUNTRIES]
# Sort choices list by country name
COUNTRY_CHOICES = sorted(COUNTRY_CHOICES, key=lambda choice: choice[1])
for country, label in COUNTRY_CHOICES:
country_rules = i18naddress.get_validation_rules({'country_code': country})
COUNTRY_FORMS[country] = construct_address_form(country, country_rules)
from shoop_tests.utils import apply_request_middleware
from .image_generator import generate_image
DEFAULT_IDENTIFIER = "default"
DEFAULT_NAME = "Default"
DEFAULT_ADDRESS_DATA = dict(prefix="Sir",
name=u"Dog Hello",
suffix=", Esq.",
postal_code="K9N",
street="Woof Ave.",
city="Dog Fort",
country="GB")
COUNTRY_CODES = sorted(COUNTRIES.keys())
class FuzzyBoolean(fuzzy.BaseFuzzyAttribute):
def __init__(self, probability, **kwargs):
self.probability = probability
super(FuzzyBoolean, self).__init__()
def fuzz(self):
return (random.random() < self.probability)
class UserFactory(DjangoModelFactory):
class Meta:
model = settings.AUTH_USER_MODEL
from .image_generator import generate_image
DEFAULT_IDENTIFIER = "default"
DEFAULT_NAME = "Default"
DEFAULT_ADDRESS_DATA = dict(
prefix="Sir",
name=u"Dog Hello",
suffix=", Esq.",
postal_code="K9N",
street="Woof Ave.",
city="Dog Fort",
country="GB"
)
COUNTRY_CODES = sorted(COUNTRIES.keys())
class FuzzyBoolean(fuzzy.BaseFuzzyAttribute):
def __init__(self, probability, **kwargs):
self.probability = probability
super(FuzzyBoolean, self).__init__()
def fuzz(self):
return (random.random() < self.probability)
class UserFactory(DjangoModelFactory):
class Meta:
model = settings.AUTH_USER_MODEL
def construct_address_form(country_code, i18n_rules):
class_name = 'AddressForm%s' % country_code
base_class = CountryAwareAddressForm
form_kwargs = {
'Meta': type(str('Meta'), (base_class.Meta, object), {}),
'formfield_callback': None}
class_ = type(base_class)(str(class_name), (base_class, ), form_kwargs)
update_base_fields(class_, i18n_rules)
class_.i18n_country_code = country_code
class_.i18n_fields_order = property(get_form_i18n_lines)
return class_
for country in COUNTRIES.keys():
try:
country_rules = i18naddress.get_validation_rules(
{'country_code': country})
except ValueError:
country_rules = i18naddress.get_validation_rules({})
UNKNOWN_COUNTRIES.add(country)
COUNTRY_CHOICES = [(code, label) for code, label in COUNTRIES.items()
if code not in UNKNOWN_COUNTRIES]
# Sort choices list by country name
COUNTRY_CHOICES = sorted(COUNTRY_CHOICES, key=lambda choice: choice[1])
for country, label in COUNTRY_CHOICES:
country_rules = i18naddress.get_validation_rules({'country_code': country})
COUNTRY_FORMS[country] = construct_address_form(country, country_rules)
def handle(self, *args, **options):
self.verbosity = options['verbosity']
self.requested_countries = update_countries = options[
'update_countries']
if not update_countries or 'ALL' in update_countries:
update_countries = (
c for c in sorted(COUNTRIES.keys())
if c >= options['start_from'] and c < options['end_before'])
self.requested_countries = list(sorted(COUNTRIES.keys()))
self.full_update = True
else:
self.full_update = False
country_result, country_removals = {}, {}
for country_code in update_countries:
# First try to retrieve the high-quality data from CommerceGuys.
result, json_data = self.load_commerceguys_data(country_code)
if not result:
country_result[country_code] = (False, False)
continue
if self.verbosity >= 2:
self.stdout.write(self.country_output_line(country_code),
ending="")
# When we have json data, use it as the main source.
if country_code == 'IR':
# Temporary solution until Google & CommerceGuys update their dataset.
json_data = None
if json_data:
region_names = self.process_commerceguys_data(
country_code, json_data)
if self.removed_regions:
country_removals[country_code] = self.removed_regions
self.print_country_summary(country_code,
region_names,
source="ComGuys")
# Retrieve country data from GeoNames.
result, geonames_data = self.load_geonames_data(
country_code, json_data is not None)
if not result:
country_result[country_code] = (json_data is not None, result)
self.update_esperanto_name_manually(country_code)
continue
# When we don't have json data, use GeoNames as the source (with caution).
if json_data is None:
region_names = self.process_geonames_data(
country_code, geonames_data)
if self.removed_regions:
country_removals[country_code] = self.removed_regions
self.print_country_summary(country_code,
region_names,
source="GeoNAPI")
# Update Esperanto names and geodata for all regions.
self.update_esperanto_name_and_bbox(country_code, json_data,
geonames_data)
if self.verbosity >= 2:
self.stdout.write("\n")
country_result[country_code] = (True, True)
# Print command execution summary.
self.print_overall_summary(country_result, (True, True),
self.style.SUCCESS,
"** Update successfully completed")
self.print_overall_summary(country_result, (False, None),
make_style(fg='green'), "** Update skipped")
self.print_overall_summary(
country_result, (True, False), self.style.WARNING,
"** Update partially done (no GeoNames data)")
self.print_overall_summary(country_result, (False, False),
self.style.ERROR, "** Update failed")
if any(country_removals.values()):
if self.verbosity >= 1:
self.stdout.write(make_style(fg='red')(
"** Some regions of {} were removed!".format(", ".join(
country_removals.keys()))),
ending=" ")
self.stdout.write(
"Run the {} command to review the invalidated "
"addresses in the database.".format(
make_style(
opts=('underscore', ))("check_country_regions")))
if self.verbosity == 2:
self.stdout.write(
make_style(fg='red')(json.dumps(country_removals,
indent=3,
ensure_ascii=False)))
def handle(self, *args, **options):
self.verbosity = options['verbosity']
update_countries = options['update_countries']
if not update_countries or update_countries == 'ALL' or 'ALL' in update_countries:
update_countries = COUNTRIES.keys()
# Read existing local database file.
with open(COUNTRIES_GEODATA_FILE, 'r') as source_file:
file_old_contents = source_file.read()
bits = re.match('(.*\nCOUNTRIES_GEO = )(?:\{\n.*?\n\})(.*)',
file_old_contents, re.DOTALL).groups()
# Make countries dict.
country_list = OrderedDict(COUNTRIES_GEO)
update_succeeded, update_failed = [], []
name_replacements = {
'BQ': "Caribbean Netherlands",
'CG': "Congo-Brazzaville",
'CW': "Curaçao island",
'CX': "Territory of Christmas Island",
'FR': "Mainland France",
'FK': "Falkland Islands",
'KP': "North Korea",
'KR': "South Korea",
'MF': "Collectivité de Saint-Martin",
'MH': "Republic of the Marshall Islands",
'MT': "Republic of Malta",
'NL': "Mainland Netherlands",
'PS': "State of Palestine",
'VG': "British Virgin Islands",
'VI': "U.S. Virgin Islands",
'ZA': "Republic of South Africa",
}
code_replacements = {
'AS': 'US',
'CC': 'AU',
'HM': 'AU',
'PR': 'US',
}
autonomous_state_codes = {
'AW': 'NL',
'AX': 'FI',
'BQ': 'NL',
'CX': 'AU',
'YT': 'FR',
'VI': 'VI',
}
# SPECIAL CASES:
# k in ('AQ', 'AW', 'CC', 'CX', 'DJ', 'DM', 'GI', 'GP', 'HM', 'JE', 'MF', 'MO', 'PR', 'SX', 'TF', 'US', 'ZA')
# TODO Countries which cross the +180/-180 longitude cannot be displayed properly on the map.
# Mapbox is working on a solution. Verify that these countries are displayed correctly:
# 'FJ', 'KI', 'NZ', 'RU'
for country_code, i in [
(k, i) for i, k in enumerate(COUNTRIES.keys(), start=1)
if k in update_countries
]:
political_country_code = code_replacements.get(
country_code, country_code)
# For countries which cannot be geocoded normally using the OpenCage's API.
georesult = custom_calculation(country_code)
# Otherwise, attempt a strict search.
if not georesult:
georesult = geocode(
name_replacements.get(country_code,
str(COUNTRIES[country_code])),
country=autonomous_state_codes.get(country_code,
political_country_code),
multiple=True)
pointed_result_index = None
for j, res in enumerate(georesult):
if (country_code not in autonomous_state_codes and obj_type(res) == 'country') \
or (country_code in autonomous_state_codes and obj_type(res) == 'state'):
georesult.set_default_result(j)
pointed_result_index = j
break
# When not found, search the world for the name of the country.
general_result_index = None
if len(
georesult
) == 0 or pointed_result_index is None or not georesult.bbox:
georesult = geocode(str(COUNTRIES[country_code]),
multiple=True)
if len(georesult) and obj_type(
georesult) != 'country' and country_code != 'AQ':
for j, res in enumerate(georesult):
if obj_type(res) in ('country', 'state', 'county', 'island') \
and res._components.get('ISO_3166-1_alpha-2') == political_country_code:
georesult.set_default_result(j)
general_result_index = j
if obj_type(res) == 'country':
break
# Output the query and search results on screen.
if self.verbosity >= 2:
self.stdout.write(
"{index:3d}. {code} ({name})\t{result!r}".format(
index=i,
code=country_code,
name=COUNTRIES[country_code],
result=georesult))
if self.verbosity >= 3:
if country_code in code_replacements or country_code in name_replacements:
self.stdout.write("\t{code} ({name})".format(
code=political_country_code,
name=name_replacements.get(country_code,
COUNTRIES[country_code])))
if obj_type(georesult) == 'country' or \
obj_type(georesult) == 'state' and country_code in autonomous_state_codes:
style = self.style.HTTP_INFO
elif obj_type(georesult) in ('state', 'county', 'island'):
style = self.style.NOTICE
else:
style = self.style.ERROR
self.stdout.write(
style("\t-- {components}".format(
components=georesult._components)))
if general_result_index is not None:
self.stdout.write(
"\t-- [result {index} out of the returned general list]"
.format(index=general_result_index + 1))
self.stdout.write("\t-- {bbox}, center: {center}".format(
bbox=georesult.bbox, center=georesult.xy))
# Store the result in the corresponding success or failure list.
if not georesult.error and len(georesult):
geodata = {'bbox': georesult.bbox, 'center': georesult.xy}
if geodata != country_list.get(country_code, {}):
update_succeeded.append(country_code)
country_list[country_code] = geodata
else:
update_failed.append(country_code)
# Generate new local database file.
file_new_contents = bits[0]
file_new_contents += json.dumps(country_list,
indent=4).replace('null', str(None))
file_new_contents += bits[1]
with open(COUNTRIES_GEODATA_FILE, 'w') as output_file:
output_file.write(file_new_contents)
# Print command execution summary.
if self.verbosity >= 1:
if not update_failed:
self.stdout.write(
self.style.SUCCESS(
"** Geodata update successfully completed. {} countries updated."
.format(len(update_succeeded))))
if update_succeeded and self.verbosity >= 2:
self.stdout.write(
self.style.SUCCESS(" {}".format(
", ".join(update_succeeded))))
else:
self.stdout.write(
self.style.ERROR(
"** Geodata update failed. {} countries updated, {} countries failed."
.format(len(update_succeeded), len(update_failed))))
if self.verbosity >= 2:
self.stdout.write(
self.style.SUCCESS(" SUCCESS: {}".format(
", ".join(update_succeeded
) if update_succeeded else "none")))
self.stdout.write(
self.style.ERROR(" FAILURE: {}".format(
", ".join(update_failed))))
