def import_body_and_metadata(
self,
body_id: str,
importer: Importer,
userinput: str,
ags: Optional[str],
skip_body_extra: bool = False,
) -> Tuple[JSON, str]:
logger.info(f"Fetching the body {body_id}")
[body_data] = importer.load_bodies(body_id)
logger.info("Importing the body")
[body] = importer.import_bodies()
importer.converter.default_body = body
logger.info("Looking up the Amtliche Gemeindeschlüssel")
if ags:
if len(ags) != 5 and len(ags) != 8:
logger.warning(
"Your Amtlicher Gemeindeschlüssel has {} digits instead of 5 or 8"
.format(len(ags)))
body.ags = ags
else:
ags, match_name = self.get_ags(body, importer.loader.system,
userinput)
body.ags = ags
# Sometimes there's a bad short name (e.g. "Rat" for Erkelenz),
# so we use the name that's in wikidata instead
body.short_name = match_name
body.save()
logger.info("Using {} as Amtliche Gemeindeschlüssel for '{}'".format(
body.ags, body.short_name))
dotenv = ""
if body.id != settings.SITE_DEFAULT_BODY:
dotenv += f"SITE_DEFAULT_BODY={body.id}\n"
if dotenv:
logger.info(
"Found the oparl endpoint. Please add the following line to your dotenv file "
"(you'll be reminded again after the import finished): \n\n" +
dotenv)
if not skip_body_extra:
logger.info("Importing the shape of the city")
import_outline(body)
logger.info("Importing the streets")
import_streets(body)
logger.info(
f"Body {body.short_name} import with geo data successful.")
else:
logger.info(
f"Body {body.short_name} import successful. "
f"Don't forget to run import_streets, import_amenities and import_outline"
)
return body_data.data, dotenv
def handle(self, *args, **options):
cli = Cli()
userinput = options["cityname"]
body_id, entrypoint = cli.get_entrypoint_and_body(
userinput, options["mirror"])
importer = Importer(get_loader_from_system(entrypoint))
if options["manual"]:
logger.info("Fetching the body")
importer.load_bodies(body_id)
logger.info("Importing the body")
[body] = importer.import_bodies()
logger.info("The body id is {}".format(body.id))
else:
cli.import_body_and_metadata(body_id, importer, userinput,
options["ags"])
def handle(self, *args, **options):
cli = Cli()
userinput = options["cityname"]
body_id, entrypoint = cli.get_entrypoint_and_body(
userinput, options["mirror"])
importer = Importer(get_loader_from_system(entrypoint))
if options["manual"]:
if CachedObject.objects.filter(url=body_id).exists():
logger.info("Using fetched body")
else:
logger.info("Fetching the body")
importer.load_bodies(body_id)
logger.info("Importing the body")
[body] = importer.import_bodies()
logger.info(f"The body id is {body.id}")
else:
cli.import_body_and_metadata(body_id, importer, userinput,
options["ags"])
def test_missing_organization(caplog):
with RequestsMock() as requests_mock:
requests_mock.add(
requests_mock.GET,
"http://oparl.wuppertal.de/oparl/bodies/0001/organizations/gr/230",
json={"error": "not found"},
status=404,
)
# Add another one to test for uniqueness constraints
requests_mock.add(
requests_mock.GET,
"http://oparl.wuppertal.de/oparl/bodies/0001/organizations/gr/231",
json={"error": "not found"},
status=404,
)
requests_mock.add(
requests_mock.GET,
"http://oparl.wuppertal.de/oparl/bodies",
json={
"data": [
json.loads(
Path("testdata/oparl-missing/body.json").read_text())
],
"links": {},
"pagination": {},
},
)
requests_mock.add(
requests_mock.GET,
"http://oparl.wuppertal.de/oparl/bodies/0001/organizations",
json=empty_page,
)
requests_mock.add(
requests_mock.GET,
"http://oparl.wuppertal.de/oparl/bodies/0001/people",
json=empty_page,
)
requests_mock.add(
requests_mock.GET,
"http://oparl.wuppertal.de/oparl/bodies/0001/papers",
json=empty_page,
)
requests_mock.add(
requests_mock.GET,
"http://oparl.wuppertal.de/oparl/bodies/0001/meetings",
json={
"data": [
json.loads(
Path(
"testdata/oparl-missing/meeting.json").read_text())
],
"links": {},
"pagination": {},
},
)
requests_mock.add(
requests_mock.GET,
"http://oparl.wuppertal.de/oparl/bodies/0001/people/292",
status=404,
)
body_id = "http://oparl.wuppertal.de/oparl/bodies/0001"
importer = Importer(
BaseLoader(
json.loads(
Path("testdata/oparl-missing/system.json").read_text())),
force_singlethread=True,
)
[body_data] = importer.load_bodies(body_id)
[body] = importer.import_bodies()
importer.converter.default_body = body
body.ags = "05124000"
importer.fetch_lists_initial([body_data.data])
importer.import_objects()
assert set(i.oparl_id for i in Organization.objects.all()) == {
"http://oparl.wuppertal.de/oparl/bodies/0001/organizations/gr/230",
"http://oparl.wuppertal.de/oparl/bodies/0001/organizations/gr/231",
}
assert list(i.short_name for i in Organization.objects.all()) == [
"Missing",
"Missing",
]
assert Person.objects.first().name == "Missing Person"
assert caplog.messages == [
"The Person http://oparl.wuppertal.de/oparl/bodies/0001/people/292 linked "
"from http://oparl.wuppertal.de/oparl/bodies/0001/meetings/19160 was supposed "
"to be a part of the external lists, but was not. This is a bug in the OParl "
"implementation.",
"Failed to load http://oparl.wuppertal.de/oparl/bodies/0001/people/292: 404 "
"Client Error: Not Found for url: "
"http://oparl.wuppertal.de/oparl/bodies/0001/people/292",
"Using a dummy for http://oparl.wuppertal.de/oparl/bodies/0001/people/292. "
"THIS IS BAD.",
"The Organization "
"http://oparl.wuppertal.de/oparl/bodies/0001/organizations/gr/230 linked from "
"http://oparl.wuppertal.de/oparl/bodies/0001/meetings/19160 was supposed to "
"be a part of the external lists, but was not. This is a bug in the OParl "
"implementation.",
"Failed to load "
"http://oparl.wuppertal.de/oparl/bodies/0001/organizations/gr/230: 404 Client "
"Error: Not Found for url: "
"http://oparl.wuppertal.de/oparl/bodies/0001/organizations/gr/230",
"Using a dummy for "
"http://oparl.wuppertal.de/oparl/bodies/0001/organizations/gr/230. THIS IS "
"BAD.",
"The Organization "
"http://oparl.wuppertal.de/oparl/bodies/0001/organizations/gr/231 linked from "
"http://oparl.wuppertal.de/oparl/bodies/0001/meetings/19160 was supposed to "
"be a part of the external lists, but was not. This is a bug in the OParl "
"implementation.",
"Failed to load "
"http://oparl.wuppertal.de/oparl/bodies/0001/organizations/gr/231: 404 Client "
"Error: Not Found for url: "
"http://oparl.wuppertal.de/oparl/bodies/0001/organizations/gr/231",
"Using a dummy for "
"http://oparl.wuppertal.de/oparl/bodies/0001/organizations/gr/231. THIS IS "
"BAD.",
]