class WhereaboutsFactory(DjangoModelFactory): class Meta: model = 'hosting.Whereabouts' type = Faker('random_element', elements=[ch.value for ch in LocationType]) @factory.lazy_attribute def name(self): return Faker._get_faker().city().upper() @factory.lazy_attribute def state(self): if self.country in countries_with_mandatory_region(): return CountryRegionFactory(country=self.country).iso_code else: return "" country = Faker('random_element', elements=COUNTRIES.keys()) @factory.lazy_attribute def bbox(self): minx, miny, maxx, maxy = self.center.buffer(width=uniform_random( 0.05, 0.45), quadsegs=2).extent return LineString((minx, miny), (maxx, maxy), srid=SRID) @factory.lazy_attribute def center(self): # Cannot use the 'local_latlng' Faker, they don't have all countries in the database! return Point([ uniform_random(a, b) for a, b in zip(*COUNTRIES_GEO[self.country]['bbox'].values()) ], srid=SRID)
def add_arguments(self, parser): """ The command can be given a list of ISO country codes to update, as an argument. A missing argument or the word "ALL" mean that all countries in the local database will be refreshed. """ parser.add_argument( 'update_countries', nargs='*', choices=tuple(sorted(COUNTRIES.keys())) + ('ALL', ), default='ALL', metavar='country code', help= "one or more country codes to update, omit to update all countries." ) parser.add_argument( '-s', '--start-from', default='AA', choices='ABCDEFGHIJKLMNOPQRSTUVWXYZ', help="when all countries are being updated, begin with the ISO code " "starting with this letter.") parser.add_argument( '-e', '--end-before', default='ZZ', choices='ABCDEFGHIJKLMNOPQRSTUVWXYZ', help= "when all countries are being updated, stop before the ISO code " "starting with this letter.")
class TravelAdviceFactory(DjangoModelFactory): class Meta: model = 'hosting.TravelAdvice' class Params: in_past, in_present, in_future = None, None, None content = Faker('paragraph') description = factory.LazyAttribute( lambda obj: '<p>{}</p>'.format(obj.content)) countries = Faker('random_elements', elements=COUNTRIES.keys(), unique=True, length=factory.LazyFunction(lambda: randint(1, 4))) @factory.lazy_attribute def active_from(self): faker = Faker._get_faker() if self.in_past: faked_date = faker.optional_value('date_between', start_date='-365d', end_date='-200d') elif self.in_future: faked_date = faker.date_between(start_date='+2d', end_date='+199d') elif self.in_present: faked_date = faker.optional_value('date_between', start_date='-200d', end_date='-2d') else: faked_date = faker.optional_value('date_object', end_datetime='+5y') return faked_date @factory.lazy_attribute def active_until(self): faker = Faker._get_faker() if self.in_past: faked_date = faker.date_between(start_date='-199d', end_date='-2d') elif self.in_future: faked_date = faker.optional_value('date_between', start_date='+200d', end_date='+365d') elif self.in_present: faked_date = faker.optional_value('date_between', start_date='+2d', end_date='+200d') else: if self.active_from: start, end = self.active_from, self.active_from + timedelta( days=365) faked_date = faker.optional_value('date_between_dates', date_start=start, date_end=end) else: faked_date = faker.optional_value('date_object', end_datetime='+5y') return faked_date
class PlaceFactory(DjangoModelFactory): class Meta: model = 'hosting.Place' owner = factory.SubFactory('tests.factories.ProfileFactory') country = Faker('random_element', elements=COUNTRIES.keys()) @factory.lazy_attribute def state_province(self): if self.country in countries_with_mandatory_region(): region = CountryRegionFactory(country=self.country) return region.iso_code else: return "" city = Faker('city') address = Faker('address') @factory.lazy_attribute def location(self): # Cannot use the 'local_latlng' Faker, they don't have all countries in the database! return Point([ uniform_random(a, b) for a, b in zip(*COUNTRIES_GEO[self.country]['bbox'].values()) ], srid=SRID) description = Faker('paragraph', nb_sentences=4) short_description = Faker('text', max_nb_chars=140) in_book = False @staticmethod def generate_postcode(country): regex = COUNTRIES_DATA[country]['postcode_regex'] or r'\d{5}' # The * repetition qualifier makes the generator go wild, strictly limit to 1 copy. regex = regex.replace('*', '{1}') # Articially limit the length of overly permissive chunks. regex = re.sub(r'{0,\d\d}', '{0,2}', regex) # Generate a random value according to the constrained regular expression. # All whitespaces are condensed to single space and the value is uppercased. value = "" while value in ("", "GIR0AA", "GIR 0AA"): # The generator has a strong preference to this UK postal code... value = ' '.join(rstr.xeger(regex).upper().strip().split()) return value @factory.post_generation def postcode(instance, create, value, **kwargs): if not value: return if value is True: instance.postcode = PlaceFactory.generate_postcode( instance.country) else: instance.postcode = value
class CountryRegionFactory(DjangoModelFactory): class Meta: model = 'hosting.CountryRegion' class Params: short_code = factory.LazyFunction(lambda: random() < 0.20) country = Faker('random_element', elements=COUNTRIES.keys()) iso_code = Faker('pystr_format', string_format='???#', letters='ABCDEFGHJKLMNPQRSTUVWXYZ') latin_code = factory.Maybe('short_code', yes_declaration=Faker( 'pystr_format', string_format='??', letters='ABCDEFGHIJKLMNOPQRSTUVWXYZ'), no_declaration=Faker('sentence', nb_words=3)) latin_name = factory.Maybe('short_code', yes_declaration=Faker('sentence', nb_words=3), no_declaration="") @factory.lazy_attribute def esperanto_name(self): latin_region = self.latin_name or self.latin_code replacements = [ ('Q', 'Kv'), ('q', 'kv'), ('W', 'V'), ('w', 'v'), ('X', 'Ks'), ('x', 'ks'), ('Y', 'J'), ('y ', 'i '), ('y.', 'i.'), ('y', 'j'), ('Ph', 'F'), ('ph', 'f'), ('Th', 'Z'), ('th', 'z'), ('cc', 'k'), ('ee', 'i'), ('ll', 'l'), ('tt', 't'), (' ', '-'), ('.', 'o'), ] for lat_letter, esp_letter in replacements: latin_region = latin_region.replace(lat_letter, esp_letter) return latin_region
def add_arguments(self, parser): """ The command can be given a list of ISO country codes to update, as an argument. A missing argument or the word "ALL" mean that all countries in the local database will be refreshed. """ parser.add_argument( 'update_countries', nargs='*', choices=tuple(COUNTRIES.keys()) + ('ALL', ), default='ALL', metavar='country code', help= "one or more country codes to update, omit to update all countries." )
class PhoneFactory(DjangoModelFactory): class Meta: model = 'hosting.Phone' profile = factory.SubFactory('tests.factories.ProfileFactory') @factory.lazy_attribute def number(self): # the Faker's phone-number provider is a mess. phone = PhoneNumber() while not phone.is_valid(): phone = PhoneNumber(country_code=randint(1, 999), national_number=randint(10000, 99999999990)) return phone country = Faker('random_element', elements=COUNTRIES.keys()) comments = Faker('text', max_nb_chars=20) type = Faker('random_element', elements=[ch[0] for ch in PHONE_TYPE_CHOICES])
def construct_address_form(country_code, i18n_rules): class_name = 'AddressForm%s' % country_code base_class = CountryAwareAddressForm form_kwargs = { 'Meta': type(str('Meta'), (base_class.Meta, object), {}), 'formfield_callback': None } class_ = type(base_class)(str(class_name), (base_class, ), form_kwargs) update_base_fields(class_, i18n_rules) class_.i18n_country_code = country_code class_.i18n_fields_order = property(get_form_i18n_lines) return class_ for country in COUNTRIES.keys(): try: country_rules = i18naddress.get_validation_rules( {'country_code': country}) except ValueError: country_rules = i18naddress.get_validation_rules({}) UNKNOWN_COUNTRIES.add(country) COUNTRY_CHOICES = [(code, label) for code, label in COUNTRIES.items() if code not in UNKNOWN_COUNTRIES] # Sort choices list by country name COUNTRY_CHOICES = sorted(COUNTRY_CHOICES, key=lambda choice: choice[1]) for country, label in COUNTRY_CHOICES: country_rules = i18naddress.get_validation_rules({'country_code': country}) COUNTRY_FORMS[country] = construct_address_form(country, country_rules)
from shoop_tests.utils import apply_request_middleware from .image_generator import generate_image DEFAULT_IDENTIFIER = "default" DEFAULT_NAME = "Default" DEFAULT_ADDRESS_DATA = dict(prefix="Sir", name=u"Dog Hello", suffix=", Esq.", postal_code="K9N", street="Woof Ave.", city="Dog Fort", country="GB") COUNTRY_CODES = sorted(COUNTRIES.keys()) class FuzzyBoolean(fuzzy.BaseFuzzyAttribute): def __init__(self, probability, **kwargs): self.probability = probability super(FuzzyBoolean, self).__init__() def fuzz(self): return (random.random() < self.probability) class UserFactory(DjangoModelFactory): class Meta: model = settings.AUTH_USER_MODEL
from .image_generator import generate_image DEFAULT_IDENTIFIER = "default" DEFAULT_NAME = "Default" DEFAULT_ADDRESS_DATA = dict( prefix="Sir", name=u"Dog Hello", suffix=", Esq.", postal_code="K9N", street="Woof Ave.", city="Dog Fort", country="GB" ) COUNTRY_CODES = sorted(COUNTRIES.keys()) class FuzzyBoolean(fuzzy.BaseFuzzyAttribute): def __init__(self, probability, **kwargs): self.probability = probability super(FuzzyBoolean, self).__init__() def fuzz(self): return (random.random() < self.probability) class UserFactory(DjangoModelFactory): class Meta: model = settings.AUTH_USER_MODEL
def construct_address_form(country_code, i18n_rules): class_name = 'AddressForm%s' % country_code base_class = CountryAwareAddressForm form_kwargs = { 'Meta': type(str('Meta'), (base_class.Meta, object), {}), 'formfield_callback': None} class_ = type(base_class)(str(class_name), (base_class, ), form_kwargs) update_base_fields(class_, i18n_rules) class_.i18n_country_code = country_code class_.i18n_fields_order = property(get_form_i18n_lines) return class_ for country in COUNTRIES.keys(): try: country_rules = i18naddress.get_validation_rules( {'country_code': country}) except ValueError: country_rules = i18naddress.get_validation_rules({}) UNKNOWN_COUNTRIES.add(country) COUNTRY_CHOICES = [(code, label) for code, label in COUNTRIES.items() if code not in UNKNOWN_COUNTRIES] # Sort choices list by country name COUNTRY_CHOICES = sorted(COUNTRY_CHOICES, key=lambda choice: choice[1]) for country, label in COUNTRY_CHOICES: country_rules = i18naddress.get_validation_rules({'country_code': country}) COUNTRY_FORMS[country] = construct_address_form(country, country_rules)
def handle(self, *args, **options): self.verbosity = options['verbosity'] self.requested_countries = update_countries = options[ 'update_countries'] if not update_countries or 'ALL' in update_countries: update_countries = ( c for c in sorted(COUNTRIES.keys()) if c >= options['start_from'] and c < options['end_before']) self.requested_countries = list(sorted(COUNTRIES.keys())) self.full_update = True else: self.full_update = False country_result, country_removals = {}, {} for country_code in update_countries: # First try to retrieve the high-quality data from CommerceGuys. result, json_data = self.load_commerceguys_data(country_code) if not result: country_result[country_code] = (False, False) continue if self.verbosity >= 2: self.stdout.write(self.country_output_line(country_code), ending="") # When we have json data, use it as the main source. if country_code == 'IR': # Temporary solution until Google & CommerceGuys update their dataset. json_data = None if json_data: region_names = self.process_commerceguys_data( country_code, json_data) if self.removed_regions: country_removals[country_code] = self.removed_regions self.print_country_summary(country_code, region_names, source="ComGuys") # Retrieve country data from GeoNames. result, geonames_data = self.load_geonames_data( country_code, json_data is not None) if not result: country_result[country_code] = (json_data is not None, result) self.update_esperanto_name_manually(country_code) continue # When we don't have json data, use GeoNames as the source (with caution). if json_data is None: region_names = self.process_geonames_data( country_code, geonames_data) if self.removed_regions: country_removals[country_code] = self.removed_regions self.print_country_summary(country_code, region_names, source="GeoNAPI") # Update Esperanto names and geodata for all regions. self.update_esperanto_name_and_bbox(country_code, json_data, geonames_data) if self.verbosity >= 2: self.stdout.write("\n") country_result[country_code] = (True, True) # Print command execution summary. self.print_overall_summary(country_result, (True, True), self.style.SUCCESS, "** Update successfully completed") self.print_overall_summary(country_result, (False, None), make_style(fg='green'), "** Update skipped") self.print_overall_summary( country_result, (True, False), self.style.WARNING, "** Update partially done (no GeoNames data)") self.print_overall_summary(country_result, (False, False), self.style.ERROR, "** Update failed") if any(country_removals.values()): if self.verbosity >= 1: self.stdout.write(make_style(fg='red')( "** Some regions of {} were removed!".format(", ".join( country_removals.keys()))), ending=" ") self.stdout.write( "Run the {} command to review the invalidated " "addresses in the database.".format( make_style( opts=('underscore', ))("check_country_regions"))) if self.verbosity == 2: self.stdout.write( make_style(fg='red')(json.dumps(country_removals, indent=3, ensure_ascii=False)))
def handle(self, *args, **options): self.verbosity = options['verbosity'] update_countries = options['update_countries'] if not update_countries or update_countries == 'ALL' or 'ALL' in update_countries: update_countries = COUNTRIES.keys() # Read existing local database file. with open(COUNTRIES_GEODATA_FILE, 'r') as source_file: file_old_contents = source_file.read() bits = re.match('(.*\nCOUNTRIES_GEO = )(?:\{\n.*?\n\})(.*)', file_old_contents, re.DOTALL).groups() # Make countries dict. country_list = OrderedDict(COUNTRIES_GEO) update_succeeded, update_failed = [], [] name_replacements = { 'BQ': "Caribbean Netherlands", 'CG': "Congo-Brazzaville", 'CW': "Curaçao island", 'CX': "Territory of Christmas Island", 'FR': "Mainland France", 'FK': "Falkland Islands", 'KP': "North Korea", 'KR': "South Korea", 'MF': "Collectivité de Saint-Martin", 'MH': "Republic of the Marshall Islands", 'MT': "Republic of Malta", 'NL': "Mainland Netherlands", 'PS': "State of Palestine", 'VG': "British Virgin Islands", 'VI': "U.S. Virgin Islands", 'ZA': "Republic of South Africa", } code_replacements = { 'AS': 'US', 'CC': 'AU', 'HM': 'AU', 'PR': 'US', } autonomous_state_codes = { 'AW': 'NL', 'AX': 'FI', 'BQ': 'NL', 'CX': 'AU', 'YT': 'FR', 'VI': 'VI', } # SPECIAL CASES: # k in ('AQ', 'AW', 'CC', 'CX', 'DJ', 'DM', 'GI', 'GP', 'HM', 'JE', 'MF', 'MO', 'PR', 'SX', 'TF', 'US', 'ZA') # TODO Countries which cross the +180/-180 longitude cannot be displayed properly on the map. # Mapbox is working on a solution. Verify that these countries are displayed correctly: # 'FJ', 'KI', 'NZ', 'RU' for country_code, i in [ (k, i) for i, k in enumerate(COUNTRIES.keys(), start=1) if k in update_countries ]: political_country_code = code_replacements.get( country_code, country_code) # For countries which cannot be geocoded normally using the OpenCage's API. georesult = custom_calculation(country_code) # Otherwise, attempt a strict search. if not georesult: georesult = geocode( name_replacements.get(country_code, str(COUNTRIES[country_code])), country=autonomous_state_codes.get(country_code, political_country_code), multiple=True) pointed_result_index = None for j, res in enumerate(georesult): if (country_code not in autonomous_state_codes and obj_type(res) == 'country') \ or (country_code in autonomous_state_codes and obj_type(res) == 'state'): georesult.set_default_result(j) pointed_result_index = j break # When not found, search the world for the name of the country. general_result_index = None if len( georesult ) == 0 or pointed_result_index is None or not georesult.bbox: georesult = geocode(str(COUNTRIES[country_code]), multiple=True) if len(georesult) and obj_type( georesult) != 'country' and country_code != 'AQ': for j, res in enumerate(georesult): if obj_type(res) in ('country', 'state', 'county', 'island') \ and res._components.get('ISO_3166-1_alpha-2') == political_country_code: georesult.set_default_result(j) general_result_index = j if obj_type(res) == 'country': break # Output the query and search results on screen. if self.verbosity >= 2: self.stdout.write( "{index:3d}. {code} ({name})\t{result!r}".format( index=i, code=country_code, name=COUNTRIES[country_code], result=georesult)) if self.verbosity >= 3: if country_code in code_replacements or country_code in name_replacements: self.stdout.write("\t{code} ({name})".format( code=political_country_code, name=name_replacements.get(country_code, COUNTRIES[country_code]))) if obj_type(georesult) == 'country' or \ obj_type(georesult) == 'state' and country_code in autonomous_state_codes: style = self.style.HTTP_INFO elif obj_type(georesult) in ('state', 'county', 'island'): style = self.style.NOTICE else: style = self.style.ERROR self.stdout.write( style("\t-- {components}".format( components=georesult._components))) if general_result_index is not None: self.stdout.write( "\t-- [result {index} out of the returned general list]" .format(index=general_result_index + 1)) self.stdout.write("\t-- {bbox}, center: {center}".format( bbox=georesult.bbox, center=georesult.xy)) # Store the result in the corresponding success or failure list. if not georesult.error and len(georesult): geodata = {'bbox': georesult.bbox, 'center': georesult.xy} if geodata != country_list.get(country_code, {}): update_succeeded.append(country_code) country_list[country_code] = geodata else: update_failed.append(country_code) # Generate new local database file. file_new_contents = bits[0] file_new_contents += json.dumps(country_list, indent=4).replace('null', str(None)) file_new_contents += bits[1] with open(COUNTRIES_GEODATA_FILE, 'w') as output_file: output_file.write(file_new_contents) # Print command execution summary. if self.verbosity >= 1: if not update_failed: self.stdout.write( self.style.SUCCESS( "** Geodata update successfully completed. {} countries updated." .format(len(update_succeeded)))) if update_succeeded and self.verbosity >= 2: self.stdout.write( self.style.SUCCESS(" {}".format( ", ".join(update_succeeded)))) else: self.stdout.write( self.style.ERROR( "** Geodata update failed. {} countries updated, {} countries failed." .format(len(update_succeeded), len(update_failed)))) if self.verbosity >= 2: self.stdout.write( self.style.SUCCESS(" SUCCESS: {}".format( ", ".join(update_succeeded ) if update_succeeded else "none"))) self.stdout.write( self.style.ERROR(" FAILURE: {}".format( ", ".join(update_failed))))