def test_import_header(self):
line = 'h1,h2,h3,h4'
importer = Importer(None)
importer.import_line(line)
self.assertEqual('h1', importer.headers[0])
self.assertEqual('h2', importer.headers[1])
self.assertEqual('h3', importer.headers[2])
self.assertEqual('h4', importer.headers[3])
def test_basic_import(self):
inf = StringIO(testImport)
i = Importer(self.u1)
i.readModelsFromFile(inf)
self.assertEqual(len(i.books), 3)
self.assertEqual(len(i.bookshelfs), 2)
self.assertEqual(i.bookshelfs[2], self.bs)
self.assertEqual(i.bookshelfs[1].bc_name, "Schrank")
self.assertEqual(i.bookshelfs[1].user, self.u1)
def get_importer(self, options: dict) -> Tuple[Importer, Body]:
if options.get("body"):
body = Body.objects.get(oparl_id=options["body"])
else:
body = Body.objects.get(id=settings.SITE_DEFAULT_BODY)
loader = get_loader_from_body(body.oparl_id)
importer = Importer(loader,
body,
ignore_modified=options["ignore_modified"])
importer.force_singlethread = options["force_singlethread"]
return importer, body
def test_import_valid_line(self):
header = 'h1,h2,h3,h4'
line = '0.2,3,76,-48'
importer = Importer(self.db)
importer.import_line(header)
importer.import_line(line)
result = self.db.report.find_one({'h1': 0.2})
self.assertIsNotNone(result)
self.assertEqual(result['h1'], 0.2)
self.assertEqual(result['h2'], 3)
self.assertEqual(result['h3'], 76)
self.assertEqual(result['h4'], -48)
def import_body_and_metadata(
self,
body_id: str,
importer: Importer,
userinput: str,
ags: Optional[str],
skip_body_extra: bool = False,
) -> Tuple[JSON, str]:
logger.info(f"Fetching the body {body_id}")
[body_data] = importer.load_bodies(body_id)
logger.info("Importing the body")
[body] = importer.import_bodies()
importer.converter.default_body = body
logger.info("Looking up the Amtliche Gemeindeschlüssel")
if ags:
if len(ags) != 5 and len(ags) != 8:
logger.warning(
"Your Amtlicher Gemeindeschlüssel has {} digits instead of 5 or 8"
.format(len(ags)))
body.ags = ags
else:
ags, match_name = self.get_ags(body, importer.loader.system,
userinput)
body.ags = ags
# Sometimes there's a bad short name (e.g. "Rat" for Erkelenz),
# so we use the name that's in wikidata instead
body.short_name = match_name
body.save()
logger.info("Using {} as Amtliche Gemeindeschlüssel for '{}'".format(
body.ags, body.short_name))
dotenv = ""
if body.id != settings.SITE_DEFAULT_BODY:
dotenv += f"SITE_DEFAULT_BODY={body.id}\n"
if dotenv:
logger.info(
"Found the oparl endpoint. Please add the following line to your dotenv file "
"(you'll be reminded again after the import finished): \n\n" +
dotenv)
if not skip_body_extra:
logger.info("Importing the shape of the city")
import_outline(body)
logger.info("Importing the streets")
import_streets(body)
logger.info(
f"Body {body.short_name} import with geo data successful.")
else:
logger.info(
f"Body {body.short_name} import successful. "
f"Don't forget to run import_streets, import_amenities and import_outline"
)
return body_data.data, dotenv
def check_update(self):
self.new_timestamp = (self.base_timestamp +
relativedelta(years=10)).isoformat()
self.init_mock_loader()
importer = Importer(self.loader, force_singlethread=True)
importer.update(self.body_id)
for table, count in self.tables.items():
self.assertEqual(
table.objects.count(),
count,
"{}: {} vs. {}".format(table, count, table.objects.all()),
)
self.assertEqual(File.objects.count(), 6)
def from_userinput(
self,
userinput: str,
mirror: bool,
ags: Optional[str],
skip_body_extra: bool = False,
skip_files: bool = False,
) -> None:
body_id, entrypoint = self.get_entrypoint_and_body(userinput, mirror)
importer = Importer(get_loader_from_system(entrypoint))
body_data, dotenv = self.import_body_and_metadata(
body_id, importer, userinput, ags, skip_body_extra)
logger.info("Loading the bulk data from the oparl api")
importer.fetch_lists_initial([body_data])
# Also avoid "MySQL server has gone away" errors due to timeouts
# https://stackoverflow.com/a/32720475/3549270
db.close_old_connections()
logger.info("Loading the data into the database")
importer.import_objects()
if not skip_files:
logger.info("Loading the files")
importer.load_files(fallback_city=userinput)
if dotenv:
logger.info(
f"Done! Please add the following line to your dotenv file: \n\n{dotenv}\n"
)
def check_deletion(self):
self.new_timestamp = (self.base_timestamp +
relativedelta(years=200)).isoformat()
self.delete = True
self.init_mock_loader()
importer = Importer(self.loader, force_singlethread=True)
importer.update(self.body_id)
for table, count in self.tables.items():
# It doesn't make sense if our Body was deleted
if table == Body:
continue
# We use minus one because we only deleted the top level objects
self.assertEqual(table.objects.count(), count - 1,
f"{table} {table.objects.all()}")
def handle(self, *args, **options):
cli = Cli()
userinput = options["cityname"]
body_id, entrypoint = cli.get_entrypoint_and_body(
userinput, options["mirror"])
importer = Importer(get_loader_from_system(entrypoint))
if options["manual"]:
logger.info("Fetching the body")
importer.load_bodies(body_id)
logger.info("Importing the body")
[body] = importer.import_bodies()
logger.info("The body id is {}".format(body.id))
else:
cli.import_body_and_metadata(body_id, importer, userinput,
options["ags"])
def check_ignoring_unmodified(self):
""" Check that not-modified objects are ignored """
tables_with_modified = [
Body,
Organization,
Person,
Meeting,
Paper,
File,
] # must have modified and File for #41
newer_now = timezone.now()
importer = Importer(self.loader, force_singlethread=True)
importer.update(self.body_id)
for table in tables_with_modified:
logger.debug(table.__name__)
self.assertLess(table.objects.first().modified, newer_now)
def test_file_analysis(self):
loader = MockLoader()
with open(filename, "rb") as fp:
loader.files[download_url] = (fp.read(), "application/pdf")
importer = Importer(loader, force_singlethread=True)
[body] = Body.objects.all()
importer.load_files(fallback_city=body.short_name)
[file] = File.objects.all()
self.assertEqual(file.mime_type, "application/pdf")
self.assertEqual(file.page_count, 3)
self.assertEqual(len(file.parsed_text), 10019)
self.assertEqual(file.coordinates(), [{"lat": 11.35, "lon": 142.2}])
self.assertEqual(file.person_ids(), [1])
def handle(self, *args, **options):
cli = Cli()
userinput = options["cityname"]
body_id, entrypoint = cli.get_entrypoint_and_body(
userinput, options["mirror"])
importer = Importer(get_loader_from_system(entrypoint))
if options["manual"]:
if CachedObject.objects.filter(url=body_id).exists():
logger.info("Using fetched body")
else:
logger.info("Fetching the body")
importer.load_bodies(body_id)
logger.info("Importing the body")
[body] = importer.import_bodies()
logger.info(f"The body id is {body.id}")
else:
cli.import_body_and_metadata(body_id, importer, userinput,
options["ags"])
def test_file_404(pytestconfig, caplog):
"""Check that after a file has been manually deleted, it can't get re-imported and it's gone from minio"""
url = "https://example.org/file/1"
file_id = 1
make_sample_file(file_id, url)
with responses.RequestsMock() as requests_mock:
requests_mock.add(responses.GET,
url,
status=404,
content_type="text/plain")
importer = Importer(BaseLoader({}), force_singlethread=True)
[successful, failed] = importer.load_files(sample_city.name,
update=True)
assert successful == 0 and failed == 1
assert caplog.messages == [
f"File 1: Failed to download {url}",
"1 files failed to download",
]
def handle(self, *args, **options):
input_file: Path = options["input"]
logger.info("Loading the data")
with input_file.open() as fp:
json_data = json.load(fp)
if json_data["format_version"] != format_version:
raise CommandError(
f"This version of {settings.PRODUCT_NAME} can only import json format version {format_version}, "
f"but the json file you provided is version {json_data['format_version']}"
)
ris_data: RisData = converter.structure(json_data, RisData)
body = models.Body.objects.filter(name=ris_data.meta.name).first()
if not body:
logger.info("Building the body")
if options["ags"] or ris_data.meta.ags:
ags = options["ags"] or ris_data.meta.ags
else:
ags = city_to_ags(ris_data.meta.name, False)
if not ags:
raise RuntimeError(
f"Failed to determine the Amtliche Gemeindeschlüssel for '{ris_data.meta.name}'. "
f"Please look it up yourself and specify it with `--ags`"
)
logger.info(f"The Amtliche Gemeindeschlüssel is {ags}")
body = models.Body(
name=ris_data.meta.name, short_name=ris_data.meta.name, ags=ags
)
body.save()
if not options["skip_body_extra"]:
import_outline(body)
import_streets(body)
else:
logging.info("Using existing body")
# TODO: Re-enable this after some more thorough testing
# handle_counts(ris_data, options["allow_shrinkage"])
import_data(body, ris_data)
fix_sort_date(datetime.datetime.now(tz=tz.tzlocal()))
if not options["skip_download"]:
Importer(BaseLoader(dict()), force_singlethread=True).load_files(
fallback_city=body.short_name
)
if not options["no_notify_users"]:
logger.info("Sending notifications")
NotifyUsers().notify_all()
def handle(self, *args, **options):
prefix = options["prefix"]
body = Body.objects.get(oparl_id__startswith=prefix)
loader = get_loader_from_body(body.oparl_id)
importer = Importer(loader, body)
importer.force_singlethread = options["force_singlethread"]
import_plan = [File, Paper, Consultation, AgendaItem]
for class_object in import_plan:
name = class_object.__name__
stats = class_object.objects.filter(
oparl_id__startswith=prefix).delete()
logger.info("{}: {}".format(name, stats))
CachedObject.objects.filter(
url__startswith=prefix,
oparl_type=class_object.__name__).update(to_import=True)
for type_class in import_plan:
importer.import_type(type_class)
def check_basic_import(self):
self.new_timestamp = (self.base_timestamp +
relativedelta(years=-100)).isoformat()
self.init_mock_loader()
importer = Importer(self.loader, force_singlethread=True)
importer.run(self.body_id)
now = timezone.now()
for table, count in self.tables.items():
self.assertEqual(
table.objects.count(),
count,
"{}: {} vs. {}".format(table, count, table.objects.all()),
)
self.assertLess(table.objects.first().modified, now)
self.assertEqual(File.objects.count(), 6)
# Test for #56
self.assertEqual(
Meeting.by_oparl_id(
"https://oparl.example.org/meeting/281").organizations.count(),
1,
)
def test_import_invalid_line(self):
header = 'h1,h2,h3,h4'
line1 = '0.2,3,76'
line2 = '34,2342,-98,111,223'
importer = Importer(self.db)
importer.import_line(header)
importer.import_line(line1)
result = self.db.report.find_one({'h1': 0.2})
self.assertIsNone(result)
importer.import_line(line2)
result = self.db.report.find_one({'h1': 34})
self.assertIsNone(result)
def test_update_without_change_is_ignored(self):
loader = build_mock_loader()
importer = Importer(loader, force_singlethread=True)
importer.run(self.body["id"])
[paper] = Paper.objects.all()
self.assertEqual(paper.history.count(), 1)
# The "updated" list still contains the same paper object
importer.update(self.body["id"])
[paper] = Paper.objects.all()
self.assertEqual(paper.history.count(), 1)
# Consistency check: The count is increased if there is actual change
update(loader)
importer.update(self.body["id"])
[paper] = Paper.objects.all()
self.assertEqual(paper.history.count(), 2)
def test_fetch_list_update():
loader = MockLoader()
loader.api_data["https://oparl.wuppertal.de/oparl/bodies/0001/papers"] = {
"data": [],
"links": {},
"pagination": {},
}
importer = Importer(loader)
importer.fetch_list_initial("https://oparl.wuppertal.de/oparl/bodies/0001/papers")
importer.fetch_list_update("https://oparl.wuppertal.de/oparl/bodies/0001/papers")
def import_update(body_id: Optional[str] = None,
ignore_modified: bool = False) -> None:
from importer.importer import Importer
if body_id:
bodies = Body.objects.filter(oparl_id=body_id).all()
else:
bodies = Body.objects.filter(oparl_id__isnull=False).all()
for body in bodies:
logger.info("Updating body {}: {}".format(body, body.oparl_id))
loader = get_loader_from_body(body.oparl_id)
importer = Importer(loader, body, ignore_modified=ignore_modified)
importer.update(body.oparl_id)
importer.force_singlethread = True
importer.load_files(body.short_name)
def test_manual_deletion(pytestconfig):
"""Check that after a file has been manually deleted, it can't get re-imported and it's gone from minio"""
url = "https://example.org/file/1"
file_id = 1
sample_file = File(
name="Bad File",
original_id=file_id,
url=url,
claimed_size=None,
paper_original_id=sample_paper.original_id,
)
data = RisData(sample_city, None, [], [], [sample_paper], [sample_file],
[], [], [], 2)
body = Body(name=data.meta.name,
short_name=data.meta.name,
ags=data.meta.ags)
body.save()
import_data(body, data)
with responses.RequestsMock() as requests_mock:
requests_mock.add(
responses.GET,
url,
body=Path(pytestconfig.rootdir).joinpath(
"testdata/media/file.txt").read_bytes(),
status=200,
content_type="text/plain",
)
importer = Importer(BaseLoader({}), force_singlethread=True)
[successful, failed] = importer.load_files(sample_city.name)
assert successful == 1 and failed == 0
# Ensure that the file is there
assert minio_client().get_object(minio_file_bucket, str(file_id))
assert models.File.objects.filter(pk=file_id).first()
# This is what we test
models.File.objects.get(pk=file_id).manually_delete()
with pytest.raises(MinioException):
minio_client().get_object(minio_file_bucket, str(file_id))
# Another import, to ensure that manually delete is respected
import_data(body, data)
assert not models.File.objects.filter(pk=file_id).first()
with responses.RequestsMock():
importer = Importer(BaseLoader({}), force_singlethread=True)
[successful, failed] = importer.load_files(sample_city.name)
assert successful == 0 and failed == 0
with pytest.raises(MinioException):
minio_client().get_object(minio_file_bucket, str(file_id))
def test_manual_deletion(pytestconfig, caplog):
"""Check that after a file has been manually deleted, it can't get re-imported and it's gone from minio"""
url = "https://example.org/file/1"
file_id = 1
body, data = make_sample_file(file_id, url)
with responses.RequestsMock() as requests_mock:
requests_mock.add(
responses.GET,
url,
body=pytestconfig.rootpath.joinpath(
"testdata/media/file.txt").read_bytes(),
status=200,
content_type="text/plain",
)
importer = Importer(BaseLoader({}), force_singlethread=True)
[successful, failed] = importer.load_files(sample_city.name)
assert successful == 1 and failed == 0
# Ensure that the file is there
assert minio_client().get_object(minio_file_bucket, str(file_id))
assert models.File.objects.filter(pk=file_id).first()
# This is what we test
models.File.objects.get(pk=file_id).manually_delete()
with pytest.raises(MinioException):
minio_client().get_object(minio_file_bucket, str(file_id))
# Another import, to ensure that manually delete is respected
import_data(body, data)
assert not models.File.objects.filter(pk=file_id).first()
with responses.RequestsMock():
importer = Importer(BaseLoader({}), force_singlethread=True)
[successful, failed] = importer.load_files(sample_city.name)
assert successful == 0 and failed == 0
with pytest.raises(MinioException):
minio_client().get_object(minio_file_bucket, str(file_id))
assert caplog.messages == [
"File 1 has an unknown mime type: 'text/plain'",
"File 1: Couldn't get any text",
]
def get_importer(self, options: Dict[str, Any]) -> Tuple[Importer, Body]:
if options.get("body"):
body = Body.objects.get(oparl_id=options["body"])
else:
body = Body.objects.get(id=settings.SITE_DEFAULT_BODY)
if body.oparl_id is not None:
loader = get_loader_from_body(body.oparl_id)
importer = Importer(
loader, body, ignore_modified=options["ignore_modified"]
)
else:
importer = Importer(
BaseLoader(dict()), ignore_modified=options["ignore_modified"]
)
importer.force_singlethread = options["force_singlethread"]
importer.download_files = not options["skip_download"]
return importer, body
def test_embedded_update(self):
loader = build_mock_loader()
importer = Importer(loader, force_singlethread=True)
importer.run(self.body["id"])
paper_id = make_paper([])["id"]
self.assertEqual(Paper.objects.count(), 1)
file_ids = Paper.by_oparl_id(paper_id).files.values_list("oparl_id",
flat=True)
self.assertEqual(
sorted(file_ids),
[make_file(0)["id"], make_file(1)["id"]])
self.assertEqual(File.objects.count(), 2)
update(loader)
importer.update(self.body["id"])
self.assertEqual(Paper.objects.count(), 1)
self.assertEqual(File.objects.count(), 2)
file_ids = Paper.by_oparl_id(paper_id).files.values_list("oparl_id",
flat=True)
self.assertEqual(
sorted(file_ids),
[make_file(1)["id"], make_file(2)["id"]])
self.assertEqual(File.objects_with_deleted.count(), 3)
def handle(self, *args, **options):
input_file: Path = options["input"]
logger.info("Loading the data")
with input_file.open() as fp:
json_data = json.load(fp)
if json_data["format_version"] != format_version:
raise CommandError(
f"This version of {settings.PRODUCT_NAME} can only import json format version {format_version}, "
f"but the json file you provided is version {json_data['format_version']}"
)
ris_data: RisData = converter.structure(json_data, RisData)
body = models.Body.objects.filter(name=ris_data.meta.name).first()
if not body:
logger.info("Building the body")
if options["ags"] or ris_data.meta.ags:
ags = options["ags"] or ris_data.meta.ags
else:
ags = city_to_ags(ris_data.meta.name, False)
if not ags:
raise RuntimeError(
f"Failed to determine the Amtliche Gemeindeschlüssel for '{ris_data.meta.name}'. "
f"Please look it up yourself and specify it with `--ags`"
)
logger.info(f"The Amtliche Gemeindeschlüssel is {ags}")
body = models.Body(name=ris_data.meta.name,
short_name=ris_data.meta.name,
ags=ags)
body.save()
if not options["skip_body_extra"]:
import_outline(body)
import_streets(body)
else:
logging.info("Using existing body")
# TODO: Reenable this after some more thorough testing
# handle_counts(ris_data, options["allow_shrinkage"])
flush_model(models.Paper)
self.import_papers(ris_data)
self.import_files(ris_data)
paper_id_map = make_id_map(models.Paper.objects)
file_id_map = make_id_map(models.File.objects)
flush_model(models.Paper.files.through)
self.import_paper_files(ris_data, paper_id_map, file_id_map)
flush_model(models.Organization)
self.import_organizations(body, ris_data)
self.import_meeting_locations(ris_data)
locations = dict(
models.Location.objects.values_list("description", "id"))
flush_model(models.Meeting)
self.import_meetings(ris_data, locations)
meeting_id_map = make_id_map(
models.Meeting.objects.filter(oparl_id__isnull=False))
organization_name_id_map = dict(
models.Organization.objects.values_list("name", "id"))
flush_model(models.Meeting.organizations.through)
self.import_meeting_organization(meeting_id_map,
organization_name_id_map, ris_data)
flush_model(models.Person)
self.import_persons(ris_data)
flush_model(models.Consultation)
self.import_consultations(ris_data, meeting_id_map, paper_id_map)
# We don't have original ids for all agenda items (yet?),
# so we just assume meeting x paper is unique
consultation_map = {
(a, b): c
for a, b, c in models.Consultation.objects.values_list(
"meeting_id", "paper_id", "id")
}
flush_model(models.AgendaItem)
self.import_agenda_items(ris_data, consultation_map, meeting_id_map,
paper_id_map)
flush_model(models.Membership)
self.import_memberships(ris_data)
fix_sort_date(fallback_date, datetime.datetime.now(tz=tz.tzlocal()))
# With the current bulk indexing we need to do this manually
call_command("search_index", action="populate")
if not options["skip_download"]:
Importer(BaseLoader(dict()), force_singlethread=True).load_files(
fallback_city=body.short_name)
def test_missing_organization(caplog):
with RequestsMock() as requests_mock:
requests_mock.add(
requests_mock.GET,
"http://oparl.wuppertal.de/oparl/bodies/0001/organizations/gr/230",
json={"error": "not found"},
status=404,
)
# Add another one to test for uniqueness constraints
requests_mock.add(
requests_mock.GET,
"http://oparl.wuppertal.de/oparl/bodies/0001/organizations/gr/231",
json={"error": "not found"},
status=404,
)
requests_mock.add(
requests_mock.GET,
"http://oparl.wuppertal.de/oparl/bodies",
json={
"data": [
json.loads(
Path("testdata/oparl-missing/body.json").read_text())
],
"links": {},
"pagination": {},
},
)
requests_mock.add(
requests_mock.GET,
"http://oparl.wuppertal.de/oparl/bodies/0001/organizations",
json=empty_page,
)
requests_mock.add(
requests_mock.GET,
"http://oparl.wuppertal.de/oparl/bodies/0001/people",
json=empty_page,
)
requests_mock.add(
requests_mock.GET,
"http://oparl.wuppertal.de/oparl/bodies/0001/papers",
json=empty_page,
)
requests_mock.add(
requests_mock.GET,
"http://oparl.wuppertal.de/oparl/bodies/0001/meetings",
json={
"data": [
json.loads(
Path(
"testdata/oparl-missing/meeting.json").read_text())
],
"links": {},
"pagination": {},
},
)
requests_mock.add(
requests_mock.GET,
"http://oparl.wuppertal.de/oparl/bodies/0001/people/292",
status=404,
)
body_id = "http://oparl.wuppertal.de/oparl/bodies/0001"
importer = Importer(
BaseLoader(
json.loads(
Path("testdata/oparl-missing/system.json").read_text())),
force_singlethread=True,
)
[body_data] = importer.load_bodies(body_id)
[body] = importer.import_bodies()
importer.converter.default_body = body
body.ags = "05124000"
importer.fetch_lists_initial([body_data.data])
importer.import_objects()
assert set(i.oparl_id for i in Organization.objects.all()) == {
"http://oparl.wuppertal.de/oparl/bodies/0001/organizations/gr/230",
"http://oparl.wuppertal.de/oparl/bodies/0001/organizations/gr/231",
}
assert list(i.short_name for i in Organization.objects.all()) == [
"Missing",
"Missing",
]
assert Person.objects.first().name == "Missing Person"
assert caplog.messages == [
"The Person http://oparl.wuppertal.de/oparl/bodies/0001/people/292 linked "
"from http://oparl.wuppertal.de/oparl/bodies/0001/meetings/19160 was supposed "
"to be a part of the external lists, but was not. This is a bug in the OParl "
"implementation.",
"Failed to load http://oparl.wuppertal.de/oparl/bodies/0001/people/292: 404 "
"Client Error: Not Found for url: "
"http://oparl.wuppertal.de/oparl/bodies/0001/people/292",
"Using a dummy for http://oparl.wuppertal.de/oparl/bodies/0001/people/292. "
"THIS IS BAD.",
"The Organization "
"http://oparl.wuppertal.de/oparl/bodies/0001/organizations/gr/230 linked from "
"http://oparl.wuppertal.de/oparl/bodies/0001/meetings/19160 was supposed to "
"be a part of the external lists, but was not. This is a bug in the OParl "
"implementation.",
"Failed to load "
"http://oparl.wuppertal.de/oparl/bodies/0001/organizations/gr/230: 404 Client "
"Error: Not Found for url: "
"http://oparl.wuppertal.de/oparl/bodies/0001/organizations/gr/230",
"Using a dummy for "
"http://oparl.wuppertal.de/oparl/bodies/0001/organizations/gr/230. THIS IS "
"BAD.",
"The Organization "
"http://oparl.wuppertal.de/oparl/bodies/0001/organizations/gr/231 linked from "
"http://oparl.wuppertal.de/oparl/bodies/0001/meetings/19160 was supposed to "
"be a part of the external lists, but was not. This is a bug in the OParl "
"implementation.",
"Failed to load "
"http://oparl.wuppertal.de/oparl/bodies/0001/organizations/gr/231: 404 Client "
"Error: Not Found for url: "
"http://oparl.wuppertal.de/oparl/bodies/0001/organizations/gr/231",
"Using a dummy for "
"http://oparl.wuppertal.de/oparl/bodies/0001/organizations/gr/231. THIS IS "
"BAD.",
]
def test_empty_header_values_renamed_to_unknown(self):
header = 'h1,h2,,,\n'
importer = Importer(self.db)
importer.import_line(header)
self.assertListEqual(importer.headers, ['h1', 'h2', 'Unknown0','Unknown1', 'Unknown2'])
def test_datetime_valid(self):
i = Importer(None)
s ="2015-10-04 17:19:47"
self.assertEqual(i._dt(s), datetime.datetime(2015,10,04,17,19,47))
def _dateParseDateTest(self, s, r):
i = Importer(None)
self.assertEqual(i._parseDate(s), r)
def test_datetime_garbage(self):
i = Importer(None)
self.assertEqual(i._dt("asdfer"), None)
def test_membeship_get_or_load(self):
""" Sometimes main objects are not in the external lists.
Also check that cycles (between Membership and Person are resolved) """
membership = {
"id": "https://oparl.example.org/membership/0",
"type": "https://schema.oparl.org/1.1/Membership",
"person": "https://oparl.example.org/person/1",
"organization": "https://oparl.example.org/organization/1",
"created": "2011-11-11T11:11:00+01:00",
"modified": "2012-08-16T14:05:27+02:00",
}
data = [
membership,
{
"id": "https://oparl.example.org/body/1",
"type": "https://schema.oparl.org/1.1/Body",
"system": "https://oparl.example.org/",
"shortName": "Köln",
"name": "Stadt Köln, kreisfreie Stadt",
"created": "2014-01-08T14:28:31+01:00",
"modified": "2014-01-08T14:28:31+01:00",
},
{
"id": "https://oparl.example.org/organization/1",
"type": "https://schema.oparl.org/1.1/Organization",
"body": "https://oparl.example.org/body/1",
"name": "Ausschuss für Haushalt und Finanzen",
"shortName": "Finanzausschuss",
"membership": ["https://oparl.example.org/membership/1"],
"created": "2012-07-16T00:00:00+02:00",
"modified": "2012-08-16T12:34:56+02:00",
},
{
"id": "https://oparl.example.org/person/1",
"type": "https://schema.oparl.org/1.1/Person",
"body": "https://oparl.example.org/body/1",
"name": "Prof. Dr. Max Mustermann",
"familyName": "Mustermann",
"givenName": "Max",
"membership": [
{
"id": "https://oparl.example.org/membership/1",
"type": "https://schema.oparl.org/1.1/Membership",
"organization": "https://oparl.example.org/organization/1",
"role": "Vorsitzende",
"votingRight": True,
"startDate": "2013-12-03",
}
],
"created": "2011-11-11T11:11:00+01:00",
"modified": "2012-08-16T14:05:27+02:00",
},
]
loader = MockLoader()
for oparl_object in data:
loader.api_data[oparl_object["id"]] = oparl_object
importer = Importer(loader)
importer.converter.warn_missing = False
# We need to have a body to load an organization
importer.import_anything("https://oparl.example.org/body/1")
importer.import_anything("https://oparl.example.org/membership/0")
self.assertEqual(
Membership.objects.filter(oparl_id=membership["id"]).count(), 1
)
self.assertEqual(Person.objects.count(), 1)
self.assertEqual(Organization.objects.count(), 1)