def from_userinput(
self,
userinput: str,
mirror: bool,
ags: Optional[str],
skip_body_extra: bool = False,
skip_files: bool = False,
) -> None:
body_id, entrypoint = self.get_entrypoint_and_body(userinput, mirror)
importer = Importer(get_loader_from_system(entrypoint))
body_data, dotenv = self.import_body_and_metadata(
body_id, importer, userinput, ags, skip_body_extra)
logger.info("Loading the bulk data from the oparl api")
importer.fetch_lists_initial([body_data])
# Also avoid "MySQL server has gone away" errors due to timeouts
# https://stackoverflow.com/a/32720475/3549270
db.close_old_connections()
logger.info("Loading the data into the database")
importer.import_objects()
if not skip_files:
logger.info("Loading the files")
importer.load_files(fallback_city=userinput)
if dotenv:
logger.info(
f"Done! Please add the following line to your dotenv file: \n\n{dotenv}\n"
)
def test_missing_organization(caplog):
with RequestsMock() as requests_mock:
requests_mock.add(
requests_mock.GET,
"http://oparl.wuppertal.de/oparl/bodies/0001/organizations/gr/230",
json={"error": "not found"},
status=404,
)
# Add another one to test for uniqueness constraints
requests_mock.add(
requests_mock.GET,
"http://oparl.wuppertal.de/oparl/bodies/0001/organizations/gr/231",
json={"error": "not found"},
status=404,
)
requests_mock.add(
requests_mock.GET,
"http://oparl.wuppertal.de/oparl/bodies",
json={
"data": [
json.loads(
Path("testdata/oparl-missing/body.json").read_text())
],
"links": {},
"pagination": {},
},
)
requests_mock.add(
requests_mock.GET,
"http://oparl.wuppertal.de/oparl/bodies/0001/organizations",
json=empty_page,
)
requests_mock.add(
requests_mock.GET,
"http://oparl.wuppertal.de/oparl/bodies/0001/people",
json=empty_page,
)
requests_mock.add(
requests_mock.GET,
"http://oparl.wuppertal.de/oparl/bodies/0001/papers",
json=empty_page,
)
requests_mock.add(
requests_mock.GET,
"http://oparl.wuppertal.de/oparl/bodies/0001/meetings",
json={
"data": [
json.loads(
Path(
"testdata/oparl-missing/meeting.json").read_text())
],
"links": {},
"pagination": {},
},
)
requests_mock.add(
requests_mock.GET,
"http://oparl.wuppertal.de/oparl/bodies/0001/people/292",
status=404,
)
body_id = "http://oparl.wuppertal.de/oparl/bodies/0001"
importer = Importer(
BaseLoader(
json.loads(
Path("testdata/oparl-missing/system.json").read_text())),
force_singlethread=True,
)
[body_data] = importer.load_bodies(body_id)
[body] = importer.import_bodies()
importer.converter.default_body = body
body.ags = "05124000"
importer.fetch_lists_initial([body_data.data])
importer.import_objects()
assert set(i.oparl_id for i in Organization.objects.all()) == {
"http://oparl.wuppertal.de/oparl/bodies/0001/organizations/gr/230",
"http://oparl.wuppertal.de/oparl/bodies/0001/organizations/gr/231",
}
assert list(i.short_name for i in Organization.objects.all()) == [
"Missing",
"Missing",
]
assert Person.objects.first().name == "Missing Person"
assert caplog.messages == [
"The Person http://oparl.wuppertal.de/oparl/bodies/0001/people/292 linked "
"from http://oparl.wuppertal.de/oparl/bodies/0001/meetings/19160 was supposed "
"to be a part of the external lists, but was not. This is a bug in the OParl "
"implementation.",
"Failed to load http://oparl.wuppertal.de/oparl/bodies/0001/people/292: 404 "
"Client Error: Not Found for url: "
"http://oparl.wuppertal.de/oparl/bodies/0001/people/292",
"Using a dummy for http://oparl.wuppertal.de/oparl/bodies/0001/people/292. "
"THIS IS BAD.",
"The Organization "
"http://oparl.wuppertal.de/oparl/bodies/0001/organizations/gr/230 linked from "
"http://oparl.wuppertal.de/oparl/bodies/0001/meetings/19160 was supposed to "
"be a part of the external lists, but was not. This is a bug in the OParl "
"implementation.",
"Failed to load "
"http://oparl.wuppertal.de/oparl/bodies/0001/organizations/gr/230: 404 Client "
"Error: Not Found for url: "
"http://oparl.wuppertal.de/oparl/bodies/0001/organizations/gr/230",
"Using a dummy for "
"http://oparl.wuppertal.de/oparl/bodies/0001/organizations/gr/230. THIS IS "
"BAD.",
"The Organization "
"http://oparl.wuppertal.de/oparl/bodies/0001/organizations/gr/231 linked from "
"http://oparl.wuppertal.de/oparl/bodies/0001/meetings/19160 was supposed to "
"be a part of the external lists, but was not. This is a bug in the OParl "
"implementation.",
"Failed to load "
"http://oparl.wuppertal.de/oparl/bodies/0001/organizations/gr/231: 404 Client "
"Error: Not Found for url: "
"http://oparl.wuppertal.de/oparl/bodies/0001/organizations/gr/231",
"Using a dummy for "
"http://oparl.wuppertal.de/oparl/bodies/0001/organizations/gr/231. THIS IS "
"BAD.",
]
