def import_departments(self, noop=False):
obj_list = self.pk_get('department')
syncher = ModelSyncher(Department.objects.all(), lambda obj: obj.id)
self.dept_syncher = syncher
if noop:
return
for d in obj_list:
obj = syncher.get(d['id'])
if not obj:
obj = Department(id=d['id'])
obj._changed = True
self._save_translated_field(obj, 'name', d, 'name')
if obj.abbr != d['abbr']:
obj._changed = True
obj.abbr = d['abbr']
if self.org_syncher:
org_obj = self.org_syncher.get(d['org_id'])
else:
org_obj = Organization.objects.get(id=d['org_id'])
assert org_obj
if obj.organization_id != d['org_id']:
obj._changed = True
obj.organization = org_obj
if obj._changed:
obj.save()
syncher.mark(obj)
syncher.finish()
def import_units(self):
if not getattr(self, 'org_syncher', None):
self.import_organizations(noop=True)
if not getattr(self, 'dept_syncher', None):
self.import_departments(noop=True)
if self.options['single']:
obj_id = self.options['single']
obj_list = [self.pk_get('unit', obj_id)]
queryset = Unit.objects.filter(id=obj_id)
else:
obj_list = self.pk_get('unit')
queryset = Unit.objects.all().select_related('services')
self.target_srid = settings.PROJECTION_SRID
self.bounding_box = Polygon.from_bbox(settings.BOUNDING_BOX)
self.bounding_box.set_srid(4326)
gps_srs = SpatialReference(4326)
target_srs = SpatialReference(self.target_srid)
target_to_gps_ct = CoordTransform(target_srs, gps_srs)
self.bounding_box.transform(target_to_gps_ct)
self.gps_to_target_ct = CoordTransform(gps_srs, target_srs)
syncher = ModelSyncher(queryset, lambda obj: obj.id)
for idx, info in enumerate(obj_list):
self._import_unit(syncher, info)
syncher.finish()
def import_departments(self, noop=False):
obj_list = self.pk_get('department')
syncher = ModelSyncher(Department.objects.all(), lambda obj: obj.id)
self.dept_syncher = syncher
if noop:
return
for d in obj_list:
obj = syncher.get(d['id'])
if not obj:
obj = Department(id=d['id'])
obj._changed = True
self._save_translated_field(obj, 'name', d, 'name')
if obj.abbr != d['abbr']:
obj._changed = True
obj.abbr = d['abbr']
if self.org_syncher:
org_obj = self.org_syncher.get(d['org_id'])
else:
org_obj = Organization.objects.get(id=d['org_id'])
assert org_obj
if obj.organization_id != d['org_id']:
obj._changed = True
obj.organization = org_obj
if obj._changed:
obj.save()
syncher.mark(obj)
syncher.finish()
def import_organizations(self, noop=False):
obj_list = self.pk_get('organization')
syncher = ModelSyncher(Organization.objects.all(), lambda obj: obj.id)
self.org_syncher = syncher
if noop:
return
for d in obj_list:
obj = syncher.get(d['id'])
if not obj:
obj = Organization(id=d['id'])
self._save_translated_field(obj, 'name', d, 'name')
url = d['data_source_url']
if not url.startswith('http'):
url = 'http://%s' % url
if obj.data_source_url != url:
obj._changed = True
obj.data_source_url = url
if obj._changed:
obj.save()
syncher.mark(obj)
syncher.finish()
def import_organizations(self, noop=False):
obj_list = self.pk_get('organization')
syncher = ModelSyncher(Organization.objects.all(), lambda obj: obj.id)
self.org_syncher = syncher
if noop:
return
for d in obj_list:
obj = syncher.get(d['id'])
if not obj:
obj = Organization(id=d['id'])
self._save_translated_field(obj, 'name', d, 'name')
url = d['data_source_url']
if not url.startswith('http'):
url = 'http://%s' % url
if obj.data_source_url != url:
obj._changed = True
obj.data_source_url = url
if obj._changed:
obj.save()
syncher.mark(obj)
syncher.finish()
def import_units(self):
self._load_postcodes()
self.muni_by_name = {muni.name_fi.lower(): muni for muni in Municipality.objects.all()}
if self.existing_service_ids == None or len(self.existing_service_ids) < 1:
self.existing_service_ids = set(Service.objects.values_list('id', flat=True))
if not getattr(self, 'org_syncher', None):
self.import_organizations(noop=True)
if not getattr(self, 'dept_syncher', None):
self.import_departments(noop=True)
if self.verbosity:
self.logger.info("Fetching unit connections")
connections = self.pk_get('connection')
conn_by_unit = {}
for conn in connections:
unit_id = conn['unit_id']
if unit_id not in conn_by_unit:
conn_by_unit[unit_id] = []
conn_by_unit[unit_id].append(conn)
self.accessibility_variables = {x.id: x for x in AccessibilityVariable.objects.all()}
if self.verbosity:
self.logger.info("Fetching accessibility properties")
acc_properties = self.pk_get('accessibility_property')
acc_by_unit = {}
for ap in acc_properties:
unit_id = ap['unit_id']
if unit_id not in acc_by_unit:
acc_by_unit[unit_id] = []
acc_by_unit[unit_id].append(ap)
self.target_srid = PROJECTION_SRID
self.bounding_box = Polygon.from_bbox(settings.BOUNDING_BOX)
self.bounding_box.set_srid(4326)
gps_srs = SpatialReference(4326)
target_srs = SpatialReference(self.target_srid)
target_to_gps_ct = CoordTransform(target_srs, gps_srs)
self.bounding_box.transform(target_to_gps_ct)
self.gps_to_target_ct = CoordTransform(gps_srs, target_srs)
if self.options['single']:
obj_id = self.options['single']
obj_list = [self.pk_get('unit', obj_id)]
queryset = Unit.objects.filter(id=obj_id)
else:
obj_list = self._fetch_units()
queryset = Unit.objects.filter(data_source='tprek').prefetch_related('services', 'keywords')
syncher = ModelSyncher(queryset, lambda obj: obj.id)
for idx, info in enumerate(obj_list):
conn_list = conn_by_unit.get(info['id'], [])
info['connections'] = conn_list
acp_list = acc_by_unit.get(info['id'], [])
info['accessibility_properties'] = acp_list
self._import_unit(syncher, info)
syncher.finish()
def import_services(self):
obj_list = self.pk_get('service')
syncher = ModelSyncher(Service.objects.all(), lambda obj: obj.id)
for d in obj_list:
obj = syncher.get(d['id'])
if not obj:
obj = Service(id=d['id'])
obj._changed = True
self._save_translated_field(obj, 'name', d, 'name')
if 'parent_id' in d:
parent = syncher.get(d['parent_id'])
assert parent
else:
parent = None
if obj.parent != parent:
obj.parent = parent
obj._changed = True
if obj._changed:
obj.save()
syncher.mark(obj)
syncher.finish()
def import_entrances(fetch_resource=pk_get):
obj_list = fetch_resource("entrance")
syncher = ModelSyncher(UnitEntrance.objects.all(), lambda obj: obj.id)
target_srid = PROJECTION_SRID
bounding_box = Polygon.from_bbox(settings.BOUNDING_BOX)
bounding_box.srid = 4326
gps_srs = SpatialReference(4326)
target_srs = SpatialReference(target_srid)
target_to_gps_ct = CoordTransform(target_srs, gps_srs)
bounding_box.transform(target_to_gps_ct)
gps_to_target_ct = CoordTransform(gps_srs, target_srs)
for info in sorted(obj_list, key=lambda x: x["unit_id"]):
_import_unit_entrance(
syncher,
info.copy(),
bounding_box,
gps_to_target_ct,
target_srid,
)
syncher.finish()
return syncher
def import_services(self):
srv_list = self.pk_get('service')
syncher = ModelSyncher(Service.objects.exclude(pk__in=SERVICE_IDS_TO_SKIP), lambda obj: obj.id)
self.detect_duplicate_services(srv_list)
additional_root_services = [
{
'name_fi': 'Asuminen ja kaupunkiympäristö',
'name_en': 'Housing and urban environment',
'name_sv': 'Boende och stadsmiljö',
'id': 50000
}, {
'name_fi': 'Työ, talous ja hallinto',
'name_en': 'Employment, economy and administration',
'name_sv': 'Arbete, ekonomi och förvaltning',
'id': 50001
}, {
'name_fi': 'Kulttuuri, liikunta ja vapaa-aika',
'name_en': 'Culture, sports and leisure',
'name_sv': 'Kultur, motion och fritid',
'id': 50002
}, {
'name_fi': 'Liikenne ja kartat',
'name_en': 'Traffic and maps',
'name_sv': 'Trafik och kartor',
'id': 50003
},
]
service_to_new_root = {
25298: 50000, 25142: 50000,
26098: 50001, 26300: 50001, 26244: 50001,
25622: 50002, 28128: 50002, 25954: 50002,
25554: 50003, 25476: 50003
}
dupes = []
def handle_service(d):
if d['id'] in SERVICE_IDS_TO_SKIP:
return
obj = syncher.get(d['id'])
if not obj:
obj = Service(id=d['id'])
obj._changed = True
self._save_translated_field(obj, 'name', d, 'name')
if 'identical_to' in d:
master = syncher.get(d['identical_to'])
# If the master entry hasn't been saved yet, we save the
# duplicate information later.
if not master:
dupes.append((obj, d['identical_to']))
d['identical_to'] = None
else:
d['identical_to'] = None
self._set_field(obj, 'identical_to_id', d['identical_to'])
new_root = service_to_new_root.get(d['id'])
if new_root:
d['parent_id'] = new_root
if 'parent_id' in d:
parent = syncher.get(d['parent_id'])
assert parent
else:
parent = None
if obj.parent != parent:
obj.parent = parent
obj._changed = True
self._sync_searchwords(obj, d)
if obj._changed:
obj.unit_count = obj.get_unit_count()
obj.last_modified_time = datetime.now(UTC_TIMEZONE)
obj.save()
self.services_changed = True
syncher.mark(obj)
for d in additional_root_services:
handle_service(d)
for d in srv_list:
handle_service(d)
for obj, master_id in dupes:
obj.identical_to_id = master_id
obj.save(update_fields=['identical_to'])
syncher.finish()
def import_departments(noop=False, logger=None, fetch_resource=pk_get):
obj_list = fetch_resource("department")
syncher = ModelSyncher(Department.objects.all(), lambda obj: str(obj.uuid))
# self.dept_syncher = syncher
if noop:
return syncher
for d in sorted(obj_list, key=lambda x: x["hierarchy_level"]):
obj = syncher.get(d["id"])
obj_has_changed = False
if not obj:
obj = Department(uuid=d["id"])
obj_has_changed = True
fields = ("phone", "address_zip", "oid", "organization_type",
"business_id")
fields_that_need_translation = (
"name",
"abbr",
"street_address",
"address_city",
"address_postal_full",
"www",
)
obj.uuid = d["id"]
for field in fields:
if d.get(field):
if d[field] != getattr(obj, field):
obj_has_changed = True
setattr(obj, field, d.get(field))
parent_id = d.get("parent_id")
if parent_id != obj.parent_id:
obj_has_changed = True
if parent_id is None:
obj.parent_id = None
else:
try:
parent = Department.objects.get(uuid=parent_id)
obj.parent_id = parent.id
except Department.DoesNotExist:
logger and logger.error(
"Department import: no parent with uuid {} found for {}"
.format(parent_id, d["id"]))
for field in fields_that_need_translation:
if save_translated_field(obj, field, d, field):
obj_has_changed = True
muni_code = d.get("municipality_code")
if muni_code is None:
municipality = None
if muni_code is not None:
try:
municipality = Municipality.objects.get(
division__origin_id=str(muni_code))
except Municipality.DoesNotExist:
logger and logger.error(
"No municipality with code {} for department {}".format(
muni_code, d["id"]))
if obj.municipality != municipality:
obj.municipality = municipality
obj_has_changed = True
if obj_has_changed:
obj.save()
syncher.mark(obj)
syncher.finish()
return syncher
def import_units(dept_syncher=None, fetch_only_id=None,
verbosity=True, logger=None, fetch_units=_fetch_units,
fetch_resource=pk_get):
global VERBOSITY, LOGGER, EXISTING_SERVICE_NODE_IDS, EXISTING_SERVICE_IDS
EXISTING_SERVICE_NODE_IDS = None
EXISTING_SERVICE_IDS = None
VERBOSITY = verbosity
LOGGER = logger
keyword_handler = KeywordHandler(
verbosity=verbosity, logger=logger)
if VERBOSITY and not LOGGER:
LOGGER = logging.getLogger(__name__)
muni_by_name = {muni.name_fi.lower(): muni for muni in Municipality.objects.all()}
if not dept_syncher:
dept_syncher = import_departments(noop=True)
department_id_to_uuid = dict(((k, str(v)) for k, v in Department.objects.all().values_list('id', 'uuid')))
VERBOSITY and LOGGER.info("Fetching unit connections %s" % dept_syncher)
connections = fetch_resource('connection')
conn_by_unit = defaultdict(list)
for conn in connections:
unit_id = conn['unit_id']
conn_by_unit[unit_id].append(conn)
VERBOSITY and LOGGER.info("Fetching accessibility properties")
# acc_properties = self.fetch_resource('accessibility_property', v3=True)
acc_properties = fetch_resource('accessibility_property')
acc_by_unit = defaultdict(list)
for ap in acc_properties:
unit_id = ap['unit_id']
acc_by_unit[unit_id].append(ap)
VERBOSITY and LOGGER.info("Fetching ontologyword details")
details = fetch_resource('ontologyword_details')
ontologyword_details_by_unit = defaultdict(list)
for detail in details:
unit_id = detail['unit_id']
ontologyword_details_by_unit[unit_id].append(detail)
target_srid = PROJECTION_SRID
bounding_box = Polygon.from_bbox(settings.BOUNDING_BOX)
bounding_box.set_srid(4326)
gps_srs = SpatialReference(4326)
target_srs = SpatialReference(target_srid)
target_to_gps_ct = CoordTransform(target_srs, gps_srs)
bounding_box.transform(target_to_gps_ct)
gps_to_target_ct = CoordTransform(gps_srs, target_srs)
if fetch_only_id:
obj_id = fetch_only_id
obj_list = [fetch_resource('unit', obj_id, params={'official': 'yes'})]
queryset = Unit.objects.filter(id=obj_id)
else:
obj_list = fetch_units()
queryset = Unit.objects.all().prefetch_related(
'services', 'keywords', 'service_details')
syncher = ModelSyncher(queryset, lambda obj: obj.id)
for idx, info in enumerate(obj_list):
uid = info['id']
info['connections'] = conn_by_unit.get(uid, [])
info['accessibility_properties'] = acc_by_unit.get(uid, [])
info['service_details'] = ontologyword_details_by_unit.get(uid, [])
_import_unit(syncher, keyword_handler, info.copy(), dept_syncher, muni_by_name,
bounding_box, gps_to_target_ct, target_srid, department_id_to_uuid)
syncher.finish()
return dept_syncher, syncher
def _import_unit_accessibility_properties(self):
property_syncher = ModelSyncher(
UnitAccessibilityProperty.objects.all(), lambda obj: obj.id)
num_of_imports = 0
# For caching unit ids that are not present in the database
unit_skip_list = set([])
accessibility_properties = get_ar_servicepoint_accessibility_resource(
"properties")
for accessibility_property in accessibility_properties:
# Make sure that we have all the necessary property attributes
ptv_id = accessibility_property.get("servicePointId")
accessibility_variable_id = accessibility_property.get(
"variableId")
accessibility_variable_value = accessibility_property.get("value")
if not (ptv_id and accessibility_variable_id
and accessibility_variable_value):
continue
# No need to check further if the unit has already been marked as non-existing
if ptv_id in unit_skip_list:
continue
# Make sure that the unit exists
try:
# TODO: Optimize this if it gets too slow
# One way is to get all unit ids in one go and make a lookup table
unit_identifier = UnitIdentifier.objects.get(namespace="ptv",
value=ptv_id)
unit = unit_identifier.unit
except UnitIdentifier.DoesNotExist:
self.logger.info(
"Unit {} does not exist, skipping".format(ptv_id))
unit_skip_list.add(ptv_id)
continue
# Make sure that the variable exists
if accessibility_variable_id not in self._accessibility_variables:
self.logger.info("No variable {}, skipping".format(ptv_id))
continue
# Create or update the property including its associated value
uap, created = UnitAccessibilityProperty.objects.update_or_create(
unit=unit,
variable_id=accessibility_variable_id,
defaults={"value": accessibility_variable_value},
)
# If an entry was updated
if not created:
# Mark it as synced
sync_uap = property_syncher.get(uap.id)
if sync_uap:
property_syncher.mark(sync_uap)
if created:
num_of_imports += 1
property_syncher.finish()
self.logger.info(
"Imported {} accessibility properties.".format(num_of_imports))
def import_units(self):
self._load_postcodes()
self.muni_by_name = {
muni.name_fi.lower(): muni
for muni in Municipality.objects.all()
}
if self.existing_service_ids == None or len(
self.existing_service_ids) < 1:
self.existing_service_ids = set(
Service.objects.values_list('id', flat=True))
if not getattr(self, 'org_syncher', None):
self.import_organizations(noop=True)
if not getattr(self, 'dept_syncher', None):
self.import_departments(noop=True)
if self.verbosity:
self.logger.info("Fetching unit connections")
connections = self.pk_get('connection')
conn_by_unit = {}
for conn in connections:
unit_id = conn['unit_id']
if unit_id not in conn_by_unit:
conn_by_unit[unit_id] = []
conn_by_unit[unit_id].append(conn)
self.accessibility_variables = {
x.id: x
for x in AccessibilityVariable.objects.all()
}
if self.verbosity:
self.logger.info("Fetching accessibility properties")
acc_properties = self.pk_get('accessibility_property')
acc_by_unit = {}
for ap in acc_properties:
unit_id = ap['unit_id']
if unit_id not in acc_by_unit:
acc_by_unit[unit_id] = []
acc_by_unit[unit_id].append(ap)
self.target_srid = PROJECTION_SRID
self.bounding_box = Polygon.from_bbox(settings.BOUNDING_BOX)
self.bounding_box.set_srid(4326)
gps_srs = SpatialReference(4326)
target_srs = SpatialReference(self.target_srid)
target_to_gps_ct = CoordTransform(target_srs, gps_srs)
self.bounding_box.transform(target_to_gps_ct)
self.gps_to_target_ct = CoordTransform(gps_srs, target_srs)
if self.options['single']:
obj_id = self.options['single']
obj_list = [self.pk_get('unit', obj_id)]
queryset = Unit.objects.filter(id=obj_id)
else:
obj_list = self._fetch_units()
queryset = Unit.objects.filter(
data_source='tprek').prefetch_related('services', 'keywords')
syncher = ModelSyncher(queryset, lambda obj: obj.id)
for idx, info in enumerate(obj_list):
conn_list = conn_by_unit.get(info['id'], [])
info['connections'] = conn_list
acp_list = acc_by_unit.get(info['id'], [])
info['accessibility_properties'] = acp_list
self._import_unit(syncher, info)
syncher.finish()
def import_services(self):
srv_list = self.pk_get('service')
syncher = ModelSyncher(
Service.objects.exclude(pk__in=SERVICE_IDS_TO_SKIP),
lambda obj: obj.id)
self.detect_duplicate_services(srv_list)
additional_root_services = [
{
'name_fi': 'Asuminen ja kaupunkiympäristö',
'name_en': 'Housing and urban environment',
'name_sv': 'Boende och stadsmiljö',
'id': 50000
},
{
'name_fi': 'Työ, talous ja hallinto',
'name_en': 'Employment, economy and administration',
'name_sv': 'Arbete, ekonomi och förvaltning',
'id': 50001
},
{
'name_fi': 'Kulttuuri, liikunta ja vapaa-aika',
'name_en': 'Culture, sports and leisure',
'name_sv': 'Kultur, motion och fritid',
'id': 50002
},
{
'name_fi': 'Liikenne ja kartat',
'name_en': 'Traffic and maps',
'name_sv': 'Trafik och kartor',
'id': 50003
},
]
service_to_new_root = {
25298: 50000,
25142: 50000,
26098: 50001,
26300: 50001,
26244: 50001,
25622: 50002,
28128: 50002,
25954: 50002,
25554: 50003,
25476: 50003
}
dupes = []
def handle_service(d):
if d['id'] in SERVICE_IDS_TO_SKIP:
return
obj = syncher.get(d['id'])
if not obj:
obj = Service(id=d['id'])
obj._changed = True
self._save_translated_field(obj, 'name', d, 'name')
if 'identical_to' in d:
master = syncher.get(d['identical_to'])
# If the master entry hasn't been saved yet, we save the
# duplicate information later.
if not master:
dupes.append((obj, d['identical_to']))
d['identical_to'] = None
else:
d['identical_to'] = None
self._set_field(obj, 'identical_to_id', d['identical_to'])
new_root = service_to_new_root.get(d['id'])
if new_root:
d['parent_id'] = new_root
if 'parent_id' in d:
parent = syncher.get(d['parent_id'])
assert parent
else:
parent = None
if obj.parent != parent:
obj.parent = parent
obj._changed = True
self._sync_searchwords(obj, d)
if obj._changed:
obj.unit_count = obj.get_unit_count()
obj.last_modified_time = datetime.now(UTC_TIMEZONE)
obj.save()
self.services_changed = True
syncher.mark(obj)
for d in additional_root_services:
handle_service(d)
for d in srv_list:
handle_service(d)
for obj, master_id in dupes:
obj.identical_to_id = master_id
obj.save(update_fields=['identical_to'])
syncher.finish()
def import_units(dept_syncher=None,
fetch_only_id=None,
verbosity=True,
logger=None,
fetch_units=_fetch_units,
fetch_resource=pk_get):
global VERBOSITY, LOGGER, EXISTING_SERVICE_NODE_IDS, EXISTING_SERVICE_IDS
EXISTING_SERVICE_NODE_IDS = None
EXISTING_SERVICE_IDS = None
VERBOSITY = verbosity
LOGGER = logger
keyword_handler = KeywordHandler(verbosity=verbosity, logger=logger)
if VERBOSITY and not LOGGER:
LOGGER = logging.getLogger(__name__)
muni_by_name = {
muni.name_fi.lower(): muni
for muni in Municipality.objects.all()
}
if not dept_syncher:
dept_syncher = import_departments(noop=True)
department_id_to_uuid = dict(
((k, str(v))
for k, v in Department.objects.all().values_list('id', 'uuid')))
VERBOSITY and LOGGER.info("Fetching unit connections %s" % dept_syncher)
connections = fetch_resource('connection')
conn_by_unit = defaultdict(list)
for conn in connections:
unit_id = conn['unit_id']
conn_by_unit[unit_id].append(conn)
VERBOSITY and LOGGER.info("Fetching accessibility properties")
# acc_properties = self.fetch_resource('accessibility_property', v3=True)
acc_properties = fetch_resource('accessibility_property')
acc_by_unit = defaultdict(list)
for ap in acc_properties:
unit_id = ap['unit_id']
acc_by_unit[unit_id].append(ap)
VERBOSITY and LOGGER.info("Fetching ontologyword details")
details = fetch_resource('ontologyword_details')
ontologyword_details_by_unit = defaultdict(list)
for detail in details:
unit_id = detail['unit_id']
ontologyword_details_by_unit[unit_id].append(detail)
target_srid = PROJECTION_SRID
bounding_box = Polygon.from_bbox(settings.BOUNDING_BOX)
bounding_box.set_srid(4326)
gps_srs = SpatialReference(4326)
target_srs = SpatialReference(target_srid)
target_to_gps_ct = CoordTransform(target_srs, gps_srs)
bounding_box.transform(target_to_gps_ct)
gps_to_target_ct = CoordTransform(gps_srs, target_srs)
if fetch_only_id:
obj_id = fetch_only_id
obj_list = [fetch_resource('unit', obj_id, params={'official': 'yes'})]
queryset = Unit.objects.filter(id=obj_id)
else:
obj_list = fetch_units()
queryset = Unit.objects.all().prefetch_related('services', 'keywords',
'service_details')
syncher = ModelSyncher(queryset, lambda obj: obj.id)
for idx, info in enumerate(obj_list):
uid = info['id']
info['connections'] = conn_by_unit.get(uid, [])
info['accessibility_properties'] = acc_by_unit.get(uid, [])
info['service_details'] = ontologyword_details_by_unit.get(uid, [])
_import_unit(syncher, keyword_handler, info.copy(), dept_syncher,
muni_by_name, bounding_box, gps_to_target_ct, target_srid,
department_id_to_uuid)
syncher.finish()
return dept_syncher, syncher
def import_services(syncher=None, noop=False, logger=None, importer=None,
ontologytrees=pk_get('ontologytree'),
ontologywords=pk_get('ontologyword')):
nodesyncher = ModelSyncher(ServiceNode.objects.all(), lambda obj: obj.id)
servicesyncher = ModelSyncher(Service.objects.all(), lambda obj: obj.id)
def save_object(obj):
if obj._changed:
obj.last_modified_time = datetime.now(UTC_TIMEZONE)
obj.save()
if importer:
importer.services_changed = True
def _build_servicetree(ontologytrees):
tree = [ot for ot in ontologytrees if not ot.get('parent_id')]
for parent_ot in tree:
_add_ot_children(parent_ot, ontologytrees)
return tree
def _add_ot_children(parent_ot, ontologytrees):
parent_ot['children'] = [ot for ot in ontologytrees if
ot.get('parent_id') == parent_ot['id']]
for child_ot in parent_ot['children']:
_add_ot_children(child_ot, ontologytrees)
def handle_service_node(d, keyword_handler):
obj = nodesyncher.get(d['id'])
if not obj:
obj = ServiceNode(id=d['id'])
obj._changed = True
if save_translated_field(obj, 'name', d, 'name'):
obj._changed = True
if 'parent_id' in d:
parent = nodesyncher.get(d['parent_id'])
assert parent
else:
parent = None
if obj.parent != parent:
obj.parent = parent
obj._changed = True
related_services_changed = False
if obj.service_reference != d.get('ontologyword_reference', None):
obj.service_reference = d.get('ontologyword_reference')
related_services_changed = True
obj._changed = True
save_object(obj)
obj._changed = keyword_handler.sync_searchwords(obj, d, obj._changed)
save_object(obj)
nodesyncher.mark(obj)
if ((related_services_changed or obj.related_services.count() == 0) and obj.service_reference is not None):
related_service_ids = set(
(id for id in SERVICE_REFERENCE_SEPARATOR.split(obj.service_reference)))
obj.related_services.set(related_service_ids)
for child_node in d['children']:
handle_service_node(child_node, keyword_handler)
def handle_service(d, keyword_handler):
obj = servicesyncher.get(d['id'])
if not obj:
obj = Service(id=d['id'])
obj._changed = True
obj._changed |= save_translated_field(obj, 'name', d, 'ontologyword')
period_enabled = d['can_add_schoolyear']
clarification_enabled = d['can_add_clarification']
obj._changed |= period_enabled != obj.period_enabled
obj._changed |= clarification_enabled != obj.clarification_enabled
obj.period_enabled = period_enabled
obj.clarification_enabled = clarification_enabled
obj._changed = keyword_handler.sync_searchwords(obj, d, obj._changed)
if obj._changed:
obj.last_modified_time = datetime.now(UTC_TIMEZONE)
obj.save()
if importer:
importer.services_changed = True
servicesyncher.mark(obj)
return obj
tree = _build_servicetree(ontologytrees)
keyword_handler = KeywordHandler(logger=logger)
for d in tree:
handle_service_node(d, keyword_handler)
nodesyncher.finish()
for d in ontologywords:
handle_service(d, keyword_handler)
servicesyncher.finish()
def import_departments(noop=False, logger=None, fetch_resource=pk_get):
obj_list = fetch_resource('department')
syncher = ModelSyncher(Department.objects.all(), lambda obj: str(obj.uuid))
# self.dept_syncher = syncher
if noop:
return syncher
for d in sorted(obj_list, key=lambda x: x['hierarchy_level']):
obj = syncher.get(d['id'])
obj_has_changed = False
if not obj:
obj = Department(uuid=d['id'])
obj_has_changed = True
fields = ('phone', 'address_zip', 'oid', 'organization_type',
'business_id')
fields_that_need_translation = ('name', 'abbr', 'street_address', 'address_city', 'address_postal_full',
'www')
obj.uuid = d['id']
for field in fields:
if d.get(field):
if d[field] != getattr(obj, field):
obj_has_changed = True
setattr(obj, field, d.get(field))
parent_id = d.get('parent_id')
if parent_id != obj.parent_id:
obj_has_changed = True
if parent_id is None:
obj.parent_id = None
else:
try:
parent = Department.objects.get(uuid=parent_id)
obj.parent_id = parent.id
except Department.DoesNotExist:
logger and logger.error(
"Department import: no parent with uuid {} found for {}".format(
parent_id, d['id'])
)
for field in fields_that_need_translation:
if save_translated_field(obj, field, d, field):
obj_has_changed = True
muni_code = d.get('municipality_code')
if muni_code is None:
municipality = None
if muni_code is not None:
try:
municipality = Municipality.objects.get(division__origin_id=str(muni_code))
except Municipality.DoesNotExist:
logger and logger.error(
"No municipality with code {} for department {}".format(
muni_code, d['id']))
if obj.municipality != municipality:
obj.municipality = municipality
obj_has_changed = True
if obj_has_changed:
obj.save()
syncher.mark(obj)
syncher.finish()
return syncher
