def process(self):
"Procède aux vérifications et à l'import de l'enregistrement"
# Création ou récupération d'un ERP existant
self._fetch_or_create_erp()
self._check_importable()
self._import_basic_erp_fields()
self._import_coordinates()
# Commune checks and normalization
self._retrieve_commune_ext()
# Strange/invalid phone numbers
if self.erp.telephone and len(self.erp.telephone) > 20:
self.erp.telephone = None
# Build metadata
self.erp.metadata = self._build_metadata()
# Prepare comment
commentaire = self._build_commentaire()
try:
# Save erp instance
self.erp.published = True
# Attach an Accessibilite to newly created Erps
if not self.erp.has_accessibilite():
accessibilite = Accessibilite(erp=self.erp)
accessibilite.commentaire = commentaire
except DataError as err:
raise RuntimeError(f"Erreur à l'enregistrement des données: {err}") from err
return self.erp, self.unpublish_reason
def import_row(self, row, **kwargs):
familles_handicaps = self.get_familles(row.get("HANDICAPS"))
fields = self.prepare_fields(row)
if not fields or self.check_existing(fields):
return
erp = Erp(**fields)
erp.save()
accessibilite = Accessibilite(
erp=erp,
labels=["th"],
labels_familles_handicap=familles_handicaps,
commentaire=
"Ces informations ont été importées depuis data.gouv.fr: https://www.data.gouv.fr/en/datasets/marque-detat-tourisme-handicap/",
)
accessibilite.save()
if erp.activite is not None:
act = f"\n {erp.activite.nom}"
else:
act = ""
print(f"ADD {erp.nom}{act} - {erp.adresse}")
return erp
def test_history_human_readable_diff(data, client):
with reversion.create_revision():
erp = Erp(
nom="test erp",
siret="52128577500016",
published=True,
geom=Point(0, 0),
)
erp.save()
accessibilite = Accessibilite(erp=erp)
accessibilite.save()
with reversion.create_revision():
erp.siret = "52128577500017"
erp.published = False
erp.geom = Point(1, 1)
erp.save()
erp.accessibilite.cheminement_ext_nombre_marches = 42
erp.accessibilite.labels = ["dpt", "th"]
erp.accessibilite.save()
history = erp.get_history()
assert len(history) == 2 # one entry per call to model .save()
erp_diff = history[0]["diff"]
assert len(erp_diff) == 3
def get_entry(field, diff_entries):
return list(filter(lambda x: x["field"] == field, diff_entries))[0]
assert get_entry("published", erp_diff)["old"] == "Oui"
assert get_entry("published", erp_diff)["new"] == "Non"
assert get_entry("geom", erp_diff)["old"] == "0.0000, 0.0000"
assert get_entry("geom", erp_diff)["new"] == "1.0000, 1.0000"
assert get_entry("siret", erp_diff)["old"] == "52128577500016"
assert get_entry("siret", erp_diff)["new"] == "52128577500017"
a11y_diff = history[1]["diff"]
assert len(a11y_diff) == 2
assert get_entry("cheminement_ext_nombre_marches",
a11y_diff)["old"] == "Vide"
assert get_entry("cheminement_ext_nombre_marches",
a11y_diff)["new"] == "42"
assert get_entry("labels", a11y_diff)["old"] == "Vide"
assert (get_entry(
"labels",
a11y_diff)["new"] == "Destination pour Tous, Tourisme & Handicap")
def process(self):
results = {
"imported": [],
"skipped": [],
"unpublished": [],
"errors": [],
}
here = os.path.abspath(
os.path.join(os.path.abspath(__file__), "..", ".."))
resource_path = (os.path.join(os.path.dirname(here), "data",
self.filepath) if self.filepath else
f"{ROOT_DATASETS_URL}/{self.id}")
for record in self.fetcher.fetch(resource_path):
erp = None
try:
mapper = self.mapper(record, self.activite, self.today)
(erp, unpublish_reason) = mapper.process()
if not erp:
self.print_char("X")
continue
with transaction.atomic():
if unpublish_reason:
erp.published = False
erp.save()
# Attach an Accessibilite to newly created Erps
if not erp.has_accessibilite():
accessibilite = Accessibilite(erp=erp)
accessibilite.save()
else:
erp.accessibilite.save()
if unpublish_reason:
self.print_char("U")
results["unpublished"].append(
f"{str(erp)}: {unpublish_reason}")
else:
self.print_char(".")
results["imported"].append(str(erp))
except SkippedRecord as skipped_reason:
self.print_char("S")
results["skipped"].append(f"{get_erp(erp)}: {skipped_reason}")
except RuntimeError as err:
self.print_char("E")
results["errors"].append(f"{get_erp(erp)}: {str(err)}")
except (TransactionManagementError, DataError,
DatabaseError) as err:
logger.error(f"Database error while importing dataset: {err}")
self.print_char("E")
results["errors"].append(f"{str(erp)}: {str(err)}")
return results
def parse_files(self):
for f in self.all_files:
self.stdout.write(f"{f}")
tree = ET.parse(f)
root = tree.getroot()
try:
data_access = self.get_access(root)
except Exception as e:
self.stdout.write(f"Access Data Error : {e}")
raise e
try:
erp = self.import_row(root)
except Exception as e:
self.stdout.write(f"ERP Data Error : {e}")
raise e
else:
if erp and data_access:
self.stdout.write(
f"Ajout de l'ERP depuis {erp.source} (id: {erp.source_id})"
)
if hasattr(erp, "pk") and erp.pk:
self.existed_erps += 1
print(f"EXIST {erp.nom} {erp.voie} {erp.commune}")
if self.force_update:
print("\tUPDATE FORCED on this ERP")
erp.save()
else:
erp.save()
print(f"ADD {erp}")
self.imported_erps += 1
if not hasattr(erp, "accessibilite"):
erp.accessibilite = Accessibilite(**data_access)
erp.accessibilite.save()
self.erps_with_access_changed += 1
def _build_a11y(self, erp, data):
data_a11y = dict()
data_a11y["transport_station_presence"] = self._get_typed_value(
data["transport_station_presence"], "nullboolean")
data_a11y["stationnement_ext_presence"] = self._get_typed_value(
data["transport_station_presence"], "nullboolean")
data_a11y["entree_balise_sonore"] = self._get_typed_value(
data["transport_station_presence"], "nullboolean")
data_a11y["entree_aide_humaine"] = self._get_typed_value(
data["transport_station_presence"], "nullboolean")
data_a11y["entree_porte_presence"] = self._get_typed_value(
data["transport_station_presence"], "nullboolean")
data_a11y["commentaire"] = self._build_comment()
return Accessibilite(erp=erp, **data_a11y)
def test_Accessibilite_has_data(data):
acc = Accessibilite(id=1337, erp=data.erp)
assert acc.has_data() is False
acc = Accessibilite(id=1337, erp=data.erp, stationnement_presence=True)
assert acc.has_data() is True
def _factory(name, value):
instance = Accessibilite(**{name: value})
form = forms.ViewAccessibiliteForm(instance=instance)
return form.get_accessibilite_data(flatten=True)
def _populate_accessibilite(self, record):
if not self.erp.has_accessibilite():
accessibilite = Accessibilite(erp=self.erp)
self.erp.accessibilite = accessibilite
self.erp.accessibilite.commentaire = self._build_comment(record)