def test_department_from_postcode(self):
# Corsica south == 2A
post_codes = ["20000", "20137", "20700"]
for post_code in post_codes:
self.assertEqual(department_from_postcode(post_code), "2A")
# Corsica north == 2B
post_codes = ["20240", "20220", "20407", "20660"]
for post_code in post_codes:
self.assertEqual(department_from_postcode(post_code), "2B")
# DOM
post_codes = ["97500", "97000", "98800", "98000"]
for post_code in post_codes:
self.assertEqual(department_from_postcode(post_code),
post_code[:3])
# Any other city
post_codes = ["13150", "30210", "17000"]
for post_code in post_codes:
self.assertEqual(department_from_postcode(post_code),
post_code[:2])
def save(self, *args, **kwargs):
already_exists = bool(self.pk)
# There is no unicity constraint on `email` at the DB level.
# It's in anticipation of other authentication methods to
# authenticate against something else, e.g. username/password.
if not already_exists and hasattr(
self, "email") and User.email_already_exists(self.email):
raise ValidationError(self.ERROR_EMAIL_ALREADY_EXISTS)
# Update department from postal code (if possible)
self.department = department_from_postcode(self.post_code)
super().save(*args, **kwargs)
def build_siae(self, main_df_row, secondary_df_row):
siret = main_df_row["siret"]
kind = secondary_df_row["kind"]
external_id = secondary_df_row["external_id"]
siae = Siae()
siae.external_id = external_id
siae.siret = siret
siae.kind = kind
siae.naf = main_df_row["naf"]
siae.source = Siae.SOURCE_ASP
siae.name = main_df_row["name"]
siae.phone = main_df_row["phone"]
if len(siae.phone) != 10: # As we have faulty 9 digits phones.
siae.phone = "" # siae.phone cannot be null in db
siae.email = "" # Do not make the authentification email public!
siae.auth_email = secondary_df_row["auth_email"]
street_num = main_df_row["street_num"]
if street_num:
street_num = int(street_num)
street_num = f"{street_num or ''} {main_df_row['street_num_extra'] or ''}"
street_name = f"{main_df_row['street_type'] or ''} {main_df_row['street_name'] or ''}"
address_line_1 = f"{street_num} {street_name}"
address_line_1 = " ".join(address_line_1.split(
)) # Replace multiple spaces by a single space.
siae.address_line_1 = address_line_1.strip()
address_line_2 = f"{main_df_row['extra1'] or ''} {main_df_row['extra2'] or ''} {main_df_row['extra3'] or ''}"
address_line_2 = " ".join(address_line_2.split(
)) # Replace multiple spaces by a single space.
siae.address_line_2 = address_line_2.strip()
# Avoid confusing case where line1 is empty and line2 is not.
if not siae.address_line_1:
siae.address_line_1 = siae.address_line_2
siae.address_line_2 = ""
siae.city = main_df_row["city"]
siae.post_code = main_df_row["zipcode"]
siae.department = department_from_postcode(siae.post_code)
if should_siae_be_created(siae):
siae = self.geocode_siae(siae)
return siae
def handle(self, dry_run=False, path_to_file=None, **options):
self.set_logger(options.get("verbosity"))
with open(path_to_file) as csvfile:
# Count lines in CSV.
reader = csv.reader(csvfile, delimiter=",")
row_count = sum(1 for row in reader)
last_progress = 0
# Reset the iterator to iterate through the reader again.
csvfile.seek(0)
for i, row in enumerate(reader):
if i == 0:
# Skip CSV header.
continue
progress = int((100 * i) / row_count)
if progress > last_progress + 5:
self.stdout.write(
f"Creating prescriber organizations… {progress}%")
last_progress = progress
# SAFIR code is mandatory for security reasons
code_safir = row[0].strip()
if not code_safir:
self.stdout.write(f"No SAFIR code provided for line {i}")
continue
assert code_safir.isdigit()
assert len(code_safir) == 5
agency_kind = row[1].strip()
name = row[2].strip()
address_line_1 = row[4].strip()
address_line_2 = row[3].strip()
post_code = row[5].strip()
city = row[6].strip()
self.logger.debug("-" * 80)
if agency_kind == "APE" or name.upper().startswith("DT"):
name = f"Pôle emploi - {name.upper()}"
else:
name = f"Pôle emploi - {agency_kind} {name.upper()}"
self.logger.debug(name)
self.logger.debug(code_safir)
self.logger.debug(address_line_1)
self.logger.debug(address_line_2)
self.logger.debug(city)
self.logger.debug(post_code)
# See address.utils.departement
department = department_from_postcode(post_code)
self.logger.debug(department)
assert department in DEPARTMENTS
existing_org = PrescriberOrganization.objects.filter(
code_safir_pole_emploi=code_safir)
if existing_org.exists():
self.stdout.write(
f"Skipping line {i} ({name}) because an organization with"
)
self.stdout.write(
f"the following SAFIR already exists: {code_safir}.")
continue
if not dry_run:
pe_kind = PrescriberOrganization.Kind.PE
org, created = PrescriberOrganization.objects.get_or_create(
code_safir_pole_emploi=code_safir,
kind=pe_kind,
defaults={
"is_authorized": True,
"name": name,
"address_line_1": address_line_1,
"address_line_2": address_line_2,
"post_code": post_code,
"city": city,
"department": department,
},
)
org.set_coords(address=org.address_line_1,
post_code=org.post_code)
org.save()
self.logger.debug("%s created.", org.display_name)
self.stdout.write("-" * 80)
self.stdout.write("Done.")
def clean(self):
if self.department != department_from_postcode(self.post_code):
raise ValidationError(
_("Le département doit correspondre au code postal."))
super().clean()
def handle(self, dry_run=False, **options):
self.set_logger(options.get("verbosity"))
with open(CSV_FILE) as csvfile:
reader = csv.reader(csvfile, delimiter=";")
for i, row in enumerate(reader):
if i == 0:
# Skip CSV header.
continue
self.logger.debug("-" * 80)
name = row[0].strip()
name = " ".join(name.split()) # Replace multiple spaces by a single space.
self.logger.debug(name)
address_line_1 = row[1].strip()
address_line_1 = " ".join(address_line_1.split())
self.logger.debug(address_line_1)
address_line_2 = row[2].strip()
address_line_2 = " ".join(address_line_2.split())
self.logger.debug(address_line_2)
city = row[3].strip()
city = " ".join(city.split())
self.logger.debug(city)
post_code = row[4].strip()
post_code = " ".join(post_code.split())
self.logger.debug(post_code)
email = row[5].strip()
self.logger.debug(email)
assert " " not in email
phone = row[6].strip().replace(" ", "")
assert len(phone) == 10
self.logger.debug(phone)
naf = row[7]
self.logger.debug(naf)
if naf:
assert len(naf) == 5
siret = row[8].strip()
self.logger.debug(siret)
assert len(siret) == 14
if siret in SEEN_SIRET:
self.stderr.write(f"Siret already seen. Skipping.")
continue
SEEN_SIRET.add(siret)
department = department_from_postcode(post_code)
self.logger.debug(department)
assert department in DEPARTMENTS
siae_info = f"{siret} {name} - {address_line_1} - {post_code} {city}."
self.logger.debug(siae_info)
if not dry_run:
siae = Siae()
siae.siret = siret
siae.naf = naf
siae.kind = Siae.KIND_GEIQ
siae.source = Siae.SOURCE_GEIQ
siae.name = name
siae.phone = phone
siae.email = email
siae.address_line_1 = address_line_1
siae.address_line_2 = address_line_2
siae.post_code = post_code
siae.city = city
siae.department = department
if siae.address_on_one_line:
geocoding_data = get_geocoding_data(siae.address_on_one_line, post_code=siae.post_code)
if not geocoding_data or geocoding_data["score"] < API_BAN_RELIABLE_MIN_SCORE:
geocoding_data = get_geocoding_data(
siae.address_on_one_line, post_code=f"{siae.post_code[:2]}000"
)
if not geocoding_data or geocoding_data["score"] < API_BAN_RELIABLE_MIN_SCORE:
geocoding_data = get_geocoding_data(siae.address_on_one_line)
if not geocoding_data or geocoding_data["score"] < API_BAN_RELIABLE_MIN_SCORE:
geocoding_data = get_geocoding_data(siae.address_line_1)
if not geocoding_data or geocoding_data["score"] < API_BAN_RELIABLE_MIN_SCORE:
geocoding_data = get_geocoding_data(siae.address_line_2)
if not geocoding_data:
self.stderr.write(f"No geocoding data found for {siae_info}")
siae.save()
continue
siae.geocoding_score = geocoding_data["score"]
# If the score is greater than API_BAN_RELIABLE_MIN_SCORE, coords are reliable:
# use data returned by the BAN API because it's better written using accents etc.
# while the source data is in all caps etc.
# Otherwise keep the old address (which is probably wrong or incomplete).
if siae.geocoding_score >= API_BAN_RELIABLE_MIN_SCORE:
siae.address_line_1 = geocoding_data["address_line_1"]
siae.city = geocoding_data["city"]
else:
self.stderr.write(f"Geocoding not reliable for {siae_info}\n{siae.address_on_one_line}")
self.logger.debug("-" * 40)
self.logger.debug(siae.address_line_1)
self.logger.debug(siae.city)
siae.coords = geocoding_data["coords"]
siae.save()
self.stdout.write("-" * 80)
self.stdout.write("Done.")