def from_userinput( self, userinput: str, mirror: bool, ags: Optional[str], skip_body_extra: bool = False, skip_files: bool = False, ) -> None: body_id, entrypoint = self.get_entrypoint_and_body(userinput, mirror) importer = Importer(get_loader_from_system(entrypoint)) body_data, dotenv = self.import_body_and_metadata( body_id, importer, userinput, ags, skip_body_extra) logger.info("Loading the bulk data from the oparl api") importer.fetch_lists_initial([body_data]) # Also avoid "MySQL server has gone away" errors due to timeouts # https://stackoverflow.com/a/32720475/3549270 db.close_old_connections() logger.info("Loading the data into the database") importer.import_objects() if not skip_files: logger.info("Loading the files") importer.load_files(fallback_city=userinput) if dotenv: logger.info( f"Done! Please add the following line to your dotenv file: \n\n{dotenv}\n" )
def test_missing_organization(caplog): with RequestsMock() as requests_mock: requests_mock.add( requests_mock.GET, "http://oparl.wuppertal.de/oparl/bodies/0001/organizations/gr/230", json={"error": "not found"}, status=404, ) # Add another one to test for uniqueness constraints requests_mock.add( requests_mock.GET, "http://oparl.wuppertal.de/oparl/bodies/0001/organizations/gr/231", json={"error": "not found"}, status=404, ) requests_mock.add( requests_mock.GET, "http://oparl.wuppertal.de/oparl/bodies", json={ "data": [ json.loads( Path("testdata/oparl-missing/body.json").read_text()) ], "links": {}, "pagination": {}, }, ) requests_mock.add( requests_mock.GET, "http://oparl.wuppertal.de/oparl/bodies/0001/organizations", json=empty_page, ) requests_mock.add( requests_mock.GET, "http://oparl.wuppertal.de/oparl/bodies/0001/people", json=empty_page, ) requests_mock.add( requests_mock.GET, "http://oparl.wuppertal.de/oparl/bodies/0001/papers", json=empty_page, ) requests_mock.add( requests_mock.GET, "http://oparl.wuppertal.de/oparl/bodies/0001/meetings", json={ "data": [ json.loads( Path( "testdata/oparl-missing/meeting.json").read_text()) ], "links": {}, "pagination": {}, }, ) requests_mock.add( requests_mock.GET, "http://oparl.wuppertal.de/oparl/bodies/0001/people/292", status=404, ) body_id = "http://oparl.wuppertal.de/oparl/bodies/0001" importer = Importer( BaseLoader( json.loads( Path("testdata/oparl-missing/system.json").read_text())), force_singlethread=True, ) [body_data] = importer.load_bodies(body_id) [body] = importer.import_bodies() importer.converter.default_body = body body.ags = "05124000" importer.fetch_lists_initial([body_data.data]) importer.import_objects() assert set(i.oparl_id for i in Organization.objects.all()) == { "http://oparl.wuppertal.de/oparl/bodies/0001/organizations/gr/230", "http://oparl.wuppertal.de/oparl/bodies/0001/organizations/gr/231", } assert list(i.short_name for i in Organization.objects.all()) == [ "Missing", "Missing", ] assert Person.objects.first().name == "Missing Person" assert caplog.messages == [ "The Person http://oparl.wuppertal.de/oparl/bodies/0001/people/292 linked " "from http://oparl.wuppertal.de/oparl/bodies/0001/meetings/19160 was supposed " "to be a part of the external lists, but was not. This is a bug in the OParl " "implementation.", "Failed to load http://oparl.wuppertal.de/oparl/bodies/0001/people/292: 404 " "Client Error: Not Found for url: " "http://oparl.wuppertal.de/oparl/bodies/0001/people/292", "Using a dummy for http://oparl.wuppertal.de/oparl/bodies/0001/people/292. " "THIS IS BAD.", "The Organization " "http://oparl.wuppertal.de/oparl/bodies/0001/organizations/gr/230 linked from " "http://oparl.wuppertal.de/oparl/bodies/0001/meetings/19160 was supposed to " "be a part of the external lists, but was not. This is a bug in the OParl " "implementation.", "Failed to load " "http://oparl.wuppertal.de/oparl/bodies/0001/organizations/gr/230: 404 Client " "Error: Not Found for url: " "http://oparl.wuppertal.de/oparl/bodies/0001/organizations/gr/230", "Using a dummy for " "http://oparl.wuppertal.de/oparl/bodies/0001/organizations/gr/230. THIS IS " "BAD.", "The Organization " "http://oparl.wuppertal.de/oparl/bodies/0001/organizations/gr/231 linked from " "http://oparl.wuppertal.de/oparl/bodies/0001/meetings/19160 was supposed to " "be a part of the external lists, but was not. This is a bug in the OParl " "implementation.", "Failed to load " "http://oparl.wuppertal.de/oparl/bodies/0001/organizations/gr/231: 404 Client " "Error: Not Found for url: " "http://oparl.wuppertal.de/oparl/bodies/0001/organizations/gr/231", "Using a dummy for " "http://oparl.wuppertal.de/oparl/bodies/0001/organizations/gr/231. THIS IS " "BAD.", ]