def test_import_units(api_client, muni_admin_div_type, resources):
def fetch_resource(name):
return resources.get(name, set())
def fetch_units():
return fetch_resource('unit')
dept_syncher = import_departments(fetch_resource=fetch_resource)
import_services(
ontologytrees=fetch_resource('ontologytree'),
ontologywords=fetch_resource('ontologyword'))
response = get(api_client, reverse('servicenode-list'))
assert_resource_synced(response, 'ontologytree', resources)
response = get(api_client, reverse('service-list'))
assert_resource_synced(response, 'ontologyword', resources)
import_units(
fetch_units=fetch_units, fetch_resource=fetch_resource,
dept_syncher=dept_syncher)
update_service_node_counts()
response = get(api_client, '{}?include=department,services'.format(reverse('unit-list')))
assert_resource_synced(response, 'unit', resources)
source_units_by_id = dict((u['id'], u) for u in resources['unit'])
ontologyword_ids_by_unit_id = dict()
ontologyword_by_id = {}
ontologytree_by_id = {}
for owd in resources['ontologyword_details']:
s = ontologyword_ids_by_unit_id.setdefault(owd['unit_id'], set())
s.add(owd['ontologyword_id'])
for ot in resources['ontologytree']:
ontologytree_by_id[ot['id']] = ot
for ow in resources['ontologyword']:
ontologyword_by_id[ow['id']] = ow
service_node_counts = {}
service_counts = {}
imported_service_details = {}
for unit in response.data['results']:
assert_unit_correctly_imported(unit, source_units_by_id.get(unit['id']),
ontologyword_ids_by_unit_id[unit['id']])
imported_service_details[unit['id']] = unit['services']
for service_node_id in unit['service_nodes']:
service_node_counts.setdefault(service_node_id, 0)
service_node_counts[service_node_id] += 1
for service in unit['services']:
service_counts[service['id']] = service_counts.get(service['id'], 0) + 1
assert_service_details_correctly_imported(
resources['ontologyword_details'], imported_service_details)
# Check unit counts in related objects
response = get(api_client, reverse('servicenode-list'))
service_nodes = response.data['results']
for service_node in service_nodes:
# Note: only the totals are currently tested
# TODO: hierarchy + municipality specific tests
assert service_node_counts.get(service_node['id'], 0) == service_node['unit_count']['total']
assert_keywords_correct(ontologytree_by_id.get(service_node['id']), service_node)
response = get(api_client, reverse('service-list'))
services = response.data['results']
for service in services:
assert service_counts.get(service['id'], 0) == service['unit_count']['total']
to = service
frm = ontologyword_by_id[service['id']]
assert to['period_enabled'] == frm['can_add_schoolyear']
assert to['clarification_enabled'] == frm['can_add_clarification']
assert_keywords_correct(ontologyword_by_id.get(service['id']), service)
def import_units(dept_syncher=None,
fetch_only_id=None,
verbosity=True,
logger=None,
fetch_units=_fetch_units,
fetch_resource=pk_get):
global VERBOSITY, LOGGER, EXISTING_SERVICE_NODE_IDS, EXISTING_SERVICE_IDS
EXISTING_SERVICE_NODE_IDS = None
EXISTING_SERVICE_IDS = None
VERBOSITY = verbosity
LOGGER = logger
keyword_handler = KeywordHandler(verbosity=verbosity, logger=logger)
if VERBOSITY and not LOGGER:
LOGGER = logging.getLogger(__name__)
muni_by_name = {
muni.name_fi.lower(): muni
for muni in Municipality.objects.all()
}
if not dept_syncher:
dept_syncher = import_departments(noop=True)
department_id_to_uuid = dict(
((k, str(v))
for k, v in Department.objects.all().values_list('id', 'uuid')))
VERBOSITY and LOGGER.info("Fetching unit connections %s" % dept_syncher)
connections = fetch_resource('connection')
conn_by_unit = defaultdict(list)
for conn in connections:
unit_id = conn['unit_id']
conn_by_unit[unit_id].append(conn)
VERBOSITY and LOGGER.info("Fetching accessibility properties")
# acc_properties = self.fetch_resource('accessibility_property', v3=True)
acc_properties = fetch_resource('accessibility_property')
acc_by_unit = defaultdict(list)
for ap in acc_properties:
unit_id = ap['unit_id']
acc_by_unit[unit_id].append(ap)
VERBOSITY and LOGGER.info("Fetching ontologyword details")
details = fetch_resource('ontologyword_details')
ontologyword_details_by_unit = defaultdict(list)
for detail in details:
unit_id = detail['unit_id']
ontologyword_details_by_unit[unit_id].append(detail)
target_srid = PROJECTION_SRID
bounding_box = Polygon.from_bbox(settings.BOUNDING_BOX)
bounding_box.set_srid(4326)
gps_srs = SpatialReference(4326)
target_srs = SpatialReference(target_srid)
target_to_gps_ct = CoordTransform(target_srs, gps_srs)
bounding_box.transform(target_to_gps_ct)
gps_to_target_ct = CoordTransform(gps_srs, target_srs)
if fetch_only_id:
obj_id = fetch_only_id
obj_list = [fetch_resource('unit', obj_id, params={'official': 'yes'})]
queryset = Unit.objects.filter(id=obj_id)
else:
obj_list = fetch_units()
queryset = Unit.objects.all().prefetch_related('services', 'keywords',
'service_details')
syncher = ModelSyncher(queryset, lambda obj: obj.id)
for idx, info in enumerate(obj_list):
uid = info['id']
info['connections'] = conn_by_unit.get(uid, [])
info['accessibility_properties'] = acc_by_unit.get(uid, [])
info['service_details'] = ontologyword_details_by_unit.get(uid, [])
_import_unit(syncher, keyword_handler, info.copy(), dept_syncher,
muni_by_name, bounding_box, gps_to_target_ct, target_srid,
department_id_to_uuid)
syncher.finish()
return dept_syncher, syncher
def import_departments(self, noop=False):
import_departments(logger=self.logger, noop=noop)
def import_departments(self, noop=False):
import_departments(logger=self.logger, noop=noop)
def import_units(dept_syncher=None, fetch_only_id=None,
verbosity=True, logger=None, fetch_units=_fetch_units,
fetch_resource=pk_get):
global VERBOSITY, LOGGER, EXISTING_SERVICE_NODE_IDS, EXISTING_SERVICE_IDS
EXISTING_SERVICE_NODE_IDS = None
EXISTING_SERVICE_IDS = None
VERBOSITY = verbosity
LOGGER = logger
keyword_handler = KeywordHandler(
verbosity=verbosity, logger=logger)
if VERBOSITY and not LOGGER:
LOGGER = logging.getLogger(__name__)
muni_by_name = {muni.name_fi.lower(): muni for muni in Municipality.objects.all()}
if not dept_syncher:
dept_syncher = import_departments(noop=True)
department_id_to_uuid = dict(((k, str(v)) for k, v in Department.objects.all().values_list('id', 'uuid')))
VERBOSITY and LOGGER.info("Fetching unit connections %s" % dept_syncher)
connections = fetch_resource('connection')
conn_by_unit = defaultdict(list)
for conn in connections:
unit_id = conn['unit_id']
conn_by_unit[unit_id].append(conn)
VERBOSITY and LOGGER.info("Fetching accessibility properties")
# acc_properties = self.fetch_resource('accessibility_property', v3=True)
acc_properties = fetch_resource('accessibility_property')
acc_by_unit = defaultdict(list)
for ap in acc_properties:
unit_id = ap['unit_id']
acc_by_unit[unit_id].append(ap)
VERBOSITY and LOGGER.info("Fetching ontologyword details")
details = fetch_resource('ontologyword_details')
ontologyword_details_by_unit = defaultdict(list)
for detail in details:
unit_id = detail['unit_id']
ontologyword_details_by_unit[unit_id].append(detail)
target_srid = PROJECTION_SRID
bounding_box = Polygon.from_bbox(settings.BOUNDING_BOX)
bounding_box.set_srid(4326)
gps_srs = SpatialReference(4326)
target_srs = SpatialReference(target_srid)
target_to_gps_ct = CoordTransform(target_srs, gps_srs)
bounding_box.transform(target_to_gps_ct)
gps_to_target_ct = CoordTransform(gps_srs, target_srs)
if fetch_only_id:
obj_id = fetch_only_id
obj_list = [fetch_resource('unit', obj_id, params={'official': 'yes'})]
queryset = Unit.objects.filter(id=obj_id)
else:
obj_list = fetch_units()
queryset = Unit.objects.all().prefetch_related(
'services', 'keywords', 'service_details')
syncher = ModelSyncher(queryset, lambda obj: obj.id)
for idx, info in enumerate(obj_list):
uid = info['id']
info['connections'] = conn_by_unit.get(uid, [])
info['accessibility_properties'] = acc_by_unit.get(uid, [])
info['service_details'] = ontologyword_details_by_unit.get(uid, [])
_import_unit(syncher, keyword_handler, info.copy(), dept_syncher, muni_by_name,
bounding_box, gps_to_target_ct, target_srid, department_id_to_uuid)
syncher.finish()
return dept_syncher, syncher
def test_import_units(api_client, muni_admin_div_type, resources):
def fetch_resource(name):
return resources.get(name, set())
def fetch_units():
return fetch_resource('unit')
dept_syncher = import_departments(fetch_resource=fetch_resource)
import_services(ontologytrees=fetch_resource('ontologytree'),
ontologywords=fetch_resource('ontologyword'))
response = get(api_client, reverse('servicenode-list'))
assert_resource_synced(response, 'ontologytree', resources)
response = get(api_client, reverse('service-list'))
assert_resource_synced(response, 'ontologyword', resources)
import_units(fetch_units=fetch_units,
fetch_resource=fetch_resource,
dept_syncher=dept_syncher)
update_service_node_counts()
response = get(
api_client,
'{}?include=department,services'.format(reverse('unit-list')))
assert_resource_synced(response, 'unit', resources)
source_units_by_id = dict((u['id'], u) for u in resources['unit'])
ontologyword_ids_by_unit_id = dict()
ontologyword_by_id = {}
ontologytree_by_id = {}
for owd in resources['ontologyword_details']:
s = ontologyword_ids_by_unit_id.setdefault(owd['unit_id'], set())
s.add(owd['ontologyword_id'])
for ot in resources['ontologytree']:
ontologytree_by_id[ot['id']] = ot
for ow in resources['ontologyword']:
ontologyword_by_id[ow['id']] = ow
service_node_counts = {}
service_counts = {}
imported_service_details = {}
for unit in response.data['results']:
assert_unit_correctly_imported(unit,
source_units_by_id.get(unit['id']),
ontologyword_ids_by_unit_id[unit['id']])
imported_service_details[unit['id']] = unit['services']
for service_node_id in unit['service_nodes']:
service_node_counts.setdefault(service_node_id, 0)
service_node_counts[service_node_id] += 1
for service in unit['services']:
service_counts[service['id']] = service_counts.get(
service['id'], 0) + 1
assert_service_details_correctly_imported(
resources['ontologyword_details'], imported_service_details)
# Check unit counts in related objects
response = get(api_client, reverse('servicenode-list'))
service_nodes = response.data['results']
for service_node in service_nodes:
# Note: only the totals are currently tested
# TODO: hierarchy + municipality specific tests
assert service_node_counts.get(
service_node['id'], 0) == service_node['unit_count']['total']
assert_keywords_correct(ontologytree_by_id.get(service_node['id']),
service_node)
response = get(api_client, reverse('service-list'))
services = response.data['results']
for service in services:
assert service_counts.get(service['id'],
0) == service['unit_count']['total']
to = service
frm = ontologyword_by_id[service['id']]
assert to['period_enabled'] == frm['can_add_schoolyear']
assert to['clarification_enabled'] == frm['can_add_clarification']
assert_keywords_correct(ontologyword_by_id.get(service['id']), service)
def test_import_units(api_client, resources):
# Manual setup: needed because of poor hypothesis-pytest integration
a, created = AdministrativeDivisionType.objects.get_or_create(
type='muni', defaults=dict(name='Municipality'))
# End of manual setup
def fetch_resource(name):
return resources.get(name, set())
def fetch_units():
return fetch_resource('unit')
dept_syncher = import_departments(fetch_resource=fetch_resource)
import_services(ontologytrees=fetch_resource('ontologytree'),
ontologywords=fetch_resource('ontologyword'))
response = get(api_client, reverse('servicenode-list'))
assert_resource_synced(response, 'ontologytree', resources)
response = get(api_client, reverse('service-list'))
assert_resource_synced(response, 'ontologyword', resources)
import_units(fetch_units=fetch_units,
fetch_resource=fetch_resource,
dept_syncher=dept_syncher)
update_service_node_counts()
update_service_counts()
response = get(
api_client,
'{}?include=department,services'.format(reverse('unit-list')))
assert_resource_synced(response, 'unit', resources)
source_units_by_id = dict((u['id'], u) for u in resources['unit'])
ontologyword_ids_by_unit_id = dict()
ontologyword_by_id = {}
ontologytree_by_id = {}
for owd in resources['ontologyword_details']:
s = ontologyword_ids_by_unit_id.setdefault(owd['unit_id'], set())
s.add(owd['ontologyword_id'])
for ot in resources['ontologytree']:
ontologytree_by_id[ot['id']] = ot
for ow in resources['ontologyword']:
ontologyword_by_id[ow['id']] = ow
service_node_counts = {}
service_counts = {}
imported_service_details = {}
for unit in response.data['results']:
assert_unit_correctly_imported(unit,
source_units_by_id.get(unit['id']),
ontologyword_ids_by_unit_id[unit['id']])
imported_service_details[unit['id']] = unit['services']
for service_node_id in unit['service_nodes']:
service_node_counts.setdefault(service_node_id, 0)
service_node_counts[service_node_id] += 1
for service in unit['services']:
service_counts[service['id']] = service_counts.get(
service['id'], 0) + 1
assert_service_details_correctly_imported(
resources['ontologyword_details'], imported_service_details)
# Check unit counts in related objects
response = get(api_client, reverse('servicenode-list'))
service_nodes = response.data['results']
for service_node in service_nodes:
# Note: only the totals are currently tested
# TODO: hierarchy + municipality specific tests
assert service_node_counts.get(
service_node['id'], 0) == service_node['unit_count']['total']
assert_keywords_correct(ontologytree_by_id.get(service_node['id']),
service_node)
response = get(api_client, reverse('service-list'))
services = response.data['results']
for service in services:
assert service_counts.get(service['id'],
0) == service['unit_count']['total']
to = service
frm = ontologyword_by_id[service['id']]
assert to['period_enabled'] == frm['can_add_schoolyear']
assert to['clarification_enabled'] == frm['can_add_clarification']
assert_keywords_correct(ontologyword_by_id.get(service['id']), service)
# Manual teardown: needed because of poor hypothesis-pytest integration
ServiceUnitCount.objects.all().delete()
ServiceNodeUnitCount.objects.all().delete()
a.delete()
def test_import_units(api_client, resources):
# Manual setup: needed because of poor hypothesis-pytest integration
a, created = AdministrativeDivisionType.objects.get_or_create(
type="muni", defaults=dict(name="Municipality"))
# End of manual setup
def fetch_resource(name):
return resources.get(name, set())
def fetch_units():
return fetch_resource("unit")
dept_syncher = import_departments(fetch_resource=fetch_resource)
import_services(
ontologytrees=fetch_resource("ontologytree"),
ontologywords=fetch_resource("ontologyword"),
)
response = get(api_client, reverse("servicenode-list"))
assert_resource_synced(response, "ontologytree", resources)
response = get(api_client, reverse("service-list"))
assert_resource_synced(response, "ontologyword", resources)
import_units(
fetch_units=fetch_units,
fetch_resource=fetch_resource,
dept_syncher=dept_syncher,
)
update_service_node_counts()
update_service_counts()
response = get(
api_client,
"{}?include=department,services".format(reverse("unit-list")))
assert_resource_synced(response, "unit", resources)
source_units_by_id = dict((u["id"], u) for u in resources["unit"])
ontologyword_ids_by_unit_id = dict()
ontologyword_by_id = {}
ontologytree_by_id = {}
for owd in resources["ontologyword_details"]:
s = ontologyword_ids_by_unit_id.setdefault(owd["unit_id"], set())
s.add(owd["ontologyword_id"])
for ot in resources["ontologytree"]:
ontologytree_by_id[ot["id"]] = ot
for ow in resources["ontologyword"]:
ontologyword_by_id[ow["id"]] = ow
service_node_counts = {}
service_counts = {}
imported_service_details = {}
for unit in response.data["results"]:
assert_unit_correctly_imported(
unit,
source_units_by_id.get(unit["id"]),
ontologyword_ids_by_unit_id[unit["id"]],
)
imported_service_details[unit["id"]] = unit["services"]
for service_node_id in unit["service_nodes"]:
service_node_counts.setdefault(service_node_id, 0)
service_node_counts[service_node_id] += 1
for service in unit["services"]:
service_counts[service["id"]] = service_counts.get(
service["id"], 0) + 1
assert_service_details_correctly_imported(
resources["ontologyword_details"], imported_service_details)
# Check unit counts in related objects
response = get(api_client, reverse("servicenode-list"))
service_nodes = response.data["results"]
for service_node in service_nodes:
# Note: only the totals are currently tested
# TODO: hierarchy + municipality specific tests
assert (service_node_counts.get(
service_node["id"], 0) == service_node["unit_count"]["total"])
assert_keywords_correct(ontologytree_by_id.get(service_node["id"]),
service_node)
response = get(api_client, reverse("service-list"))
services = response.data["results"]
for service in services:
assert service_counts.get(service["id"],
0) == service["unit_count"]["total"]
to = service
frm = ontologyword_by_id[service["id"]]
assert to["period_enabled"] == frm["can_add_schoolyear"]
assert to["clarification_enabled"] == frm["can_add_clarification"]
assert_keywords_correct(ontologyword_by_id.get(service["id"]), service)
# Manual teardown: needed because of poor hypothesis-pytest integration
ServiceUnitCount.objects.all().delete()
ServiceNodeUnitCount.objects.all().delete()
a.delete()