def import_organizations(self, noop=False):
obj_list = self.pk_get('organization')
syncher = ModelSyncher(Organization.objects.all(), lambda obj: obj.id)
self.org_syncher = syncher
if noop:
return
for d in obj_list:
obj = syncher.get(d['id'])
if not obj:
obj = Organization(id=d['id'])
self._save_translated_field(obj, 'name', d, 'name')
url = d['data_source_url']
if not url.startswith('http'):
url = 'http://%s' % url
if obj.data_source_url != url:
obj._changed = True
obj.data_source_url = url
if obj._changed:
obj.save()
syncher.mark(obj)
syncher.finish()
def import_departments(self, noop=False):
obj_list = self.pk_get('department')
syncher = ModelSyncher(Department.objects.all(), lambda obj: obj.id)
self.dept_syncher = syncher
if noop:
return
for d in obj_list:
obj = syncher.get(d['id'])
if not obj:
obj = Department(id=d['id'])
obj._changed = True
self._save_translated_field(obj, 'name', d, 'name')
if obj.abbr != d['abbr']:
obj._changed = True
obj.abbr = d['abbr']
if self.org_syncher:
org_obj = self.org_syncher.get(d['org_id'])
else:
org_obj = Organization.objects.get(id=d['org_id'])
assert org_obj
if obj.organization_id != d['org_id']:
obj._changed = True
obj.organization = org_obj
if obj._changed:
obj.save()
syncher.mark(obj)
syncher.finish()
def import_departments(self, noop=False):
obj_list = self.pk_get('department')
syncher = ModelSyncher(Department.objects.all(), lambda obj: obj.id)
self.dept_syncher = syncher
if noop:
return
for d in obj_list:
obj = syncher.get(d['id'])
if not obj:
obj = Department(id=d['id'])
obj._changed = True
self._save_translated_field(obj, 'name', d, 'name')
if obj.abbr != d['abbr']:
obj._changed = True
obj.abbr = d['abbr']
if self.org_syncher:
org_obj = self.org_syncher.get(d['org_id'])
else:
org_obj = Organization.objects.get(id=d['org_id'])
assert org_obj
if obj.organization_id != d['org_id']:
obj._changed = True
obj.organization = org_obj
if obj._changed:
obj.save()
syncher.mark(obj)
syncher.finish()
def import_organizations(self, noop=False):
obj_list = self.pk_get('organization')
syncher = ModelSyncher(Organization.objects.all(), lambda obj: obj.id)
self.org_syncher = syncher
if noop:
return
for d in obj_list:
obj = syncher.get(d['id'])
if not obj:
obj = Organization(id=d['id'])
self._save_translated_field(obj, 'name', d, 'name')
url = d['data_source_url']
if not url.startswith('http'):
url = 'http://%s' % url
if obj.data_source_url != url:
obj._changed = True
obj.data_source_url = url
if obj._changed:
obj.save()
syncher.mark(obj)
syncher.finish()
def import_services(self):
obj_list = self.pk_get('service')
syncher = ModelSyncher(Service.objects.all(), lambda obj: obj.id)
for d in obj_list:
obj = syncher.get(d['id'])
if not obj:
obj = Service(id=d['id'])
obj._changed = True
self._save_translated_field(obj, 'name', d, 'name')
if 'parent_id' in d:
parent = syncher.get(d['parent_id'])
assert parent
else:
parent = None
if obj.parent != parent:
obj.parent = parent
obj._changed = True
if obj._changed:
obj.save()
syncher.mark(obj)
syncher.finish()
def import_departments(noop=False, logger=None, fetch_resource=pk_get):
obj_list = fetch_resource("department")
syncher = ModelSyncher(Department.objects.all(), lambda obj: str(obj.uuid))
# self.dept_syncher = syncher
if noop:
return syncher
for d in sorted(obj_list, key=lambda x: x["hierarchy_level"]):
obj = syncher.get(d["id"])
obj_has_changed = False
if not obj:
obj = Department(uuid=d["id"])
obj_has_changed = True
fields = ("phone", "address_zip", "oid", "organization_type",
"business_id")
fields_that_need_translation = (
"name",
"abbr",
"street_address",
"address_city",
"address_postal_full",
"www",
)
obj.uuid = d["id"]
for field in fields:
if d.get(field):
if d[field] != getattr(obj, field):
obj_has_changed = True
setattr(obj, field, d.get(field))
parent_id = d.get("parent_id")
if parent_id != obj.parent_id:
obj_has_changed = True
if parent_id is None:
obj.parent_id = None
else:
try:
parent = Department.objects.get(uuid=parent_id)
obj.parent_id = parent.id
except Department.DoesNotExist:
logger and logger.error(
"Department import: no parent with uuid {} found for {}"
.format(parent_id, d["id"]))
for field in fields_that_need_translation:
if save_translated_field(obj, field, d, field):
obj_has_changed = True
muni_code = d.get("municipality_code")
if muni_code is None:
municipality = None
if muni_code is not None:
try:
municipality = Municipality.objects.get(
division__origin_id=str(muni_code))
except Municipality.DoesNotExist:
logger and logger.error(
"No municipality with code {} for department {}".format(
muni_code, d["id"]))
if obj.municipality != municipality:
obj.municipality = municipality
obj_has_changed = True
if obj_has_changed:
obj.save()
syncher.mark(obj)
syncher.finish()
return syncher
def _import_unit_accessibility_properties(self):
property_syncher = ModelSyncher(
UnitAccessibilityProperty.objects.all(), lambda obj: obj.id)
num_of_imports = 0
# For caching unit ids that are not present in the database
unit_skip_list = set([])
accessibility_properties = get_ar_servicepoint_accessibility_resource(
"properties")
for accessibility_property in accessibility_properties:
# Make sure that we have all the necessary property attributes
ptv_id = accessibility_property.get("servicePointId")
accessibility_variable_id = accessibility_property.get(
"variableId")
accessibility_variable_value = accessibility_property.get("value")
if not (ptv_id and accessibility_variable_id
and accessibility_variable_value):
continue
# No need to check further if the unit has already been marked as non-existing
if ptv_id in unit_skip_list:
continue
# Make sure that the unit exists
try:
# TODO: Optimize this if it gets too slow
# One way is to get all unit ids in one go and make a lookup table
unit_identifier = UnitIdentifier.objects.get(namespace="ptv",
value=ptv_id)
unit = unit_identifier.unit
except UnitIdentifier.DoesNotExist:
self.logger.info(
"Unit {} does not exist, skipping".format(ptv_id))
unit_skip_list.add(ptv_id)
continue
# Make sure that the variable exists
if accessibility_variable_id not in self._accessibility_variables:
self.logger.info("No variable {}, skipping".format(ptv_id))
continue
# Create or update the property including its associated value
uap, created = UnitAccessibilityProperty.objects.update_or_create(
unit=unit,
variable_id=accessibility_variable_id,
defaults={"value": accessibility_variable_value},
)
# If an entry was updated
if not created:
# Mark it as synced
sync_uap = property_syncher.get(uap.id)
if sync_uap:
property_syncher.mark(sync_uap)
if created:
num_of_imports += 1
property_syncher.finish()
self.logger.info(
"Imported {} accessibility properties.".format(num_of_imports))
def import_departments(noop=False, logger=None, fetch_resource=pk_get):
obj_list = fetch_resource('department')
syncher = ModelSyncher(Department.objects.all(), lambda obj: str(obj.uuid))
# self.dept_syncher = syncher
if noop:
return syncher
for d in sorted(obj_list, key=lambda x: x['hierarchy_level']):
obj = syncher.get(d['id'])
obj_has_changed = False
if not obj:
obj = Department(uuid=d['id'])
obj_has_changed = True
fields = ('phone', 'address_zip', 'oid', 'organization_type',
'business_id')
fields_that_need_translation = ('name', 'abbr', 'street_address', 'address_city', 'address_postal_full',
'www')
obj.uuid = d['id']
for field in fields:
if d.get(field):
if d[field] != getattr(obj, field):
obj_has_changed = True
setattr(obj, field, d.get(field))
parent_id = d.get('parent_id')
if parent_id != obj.parent_id:
obj_has_changed = True
if parent_id is None:
obj.parent_id = None
else:
try:
parent = Department.objects.get(uuid=parent_id)
obj.parent_id = parent.id
except Department.DoesNotExist:
logger and logger.error(
"Department import: no parent with uuid {} found for {}".format(
parent_id, d['id'])
)
for field in fields_that_need_translation:
if save_translated_field(obj, field, d, field):
obj_has_changed = True
muni_code = d.get('municipality_code')
if muni_code is None:
municipality = None
if muni_code is not None:
try:
municipality = Municipality.objects.get(division__origin_id=str(muni_code))
except Municipality.DoesNotExist:
logger and logger.error(
"No municipality with code {} for department {}".format(
muni_code, d['id']))
if obj.municipality != municipality:
obj.municipality = municipality
obj_has_changed = True
if obj_has_changed:
obj.save()
syncher.mark(obj)
syncher.finish()
return syncher
