def import_organizations(self, noop=False): obj_list = self.pk_get('organization') syncher = ModelSyncher(Organization.objects.all(), lambda obj: obj.id) self.org_syncher = syncher if noop: return for d in obj_list: obj = syncher.get(d['id']) if not obj: obj = Organization(id=d['id']) self._save_translated_field(obj, 'name', d, 'name') url = d['data_source_url'] if not url.startswith('http'): url = 'http://%s' % url if obj.data_source_url != url: obj._changed = True obj.data_source_url = url if obj._changed: obj.save() syncher.mark(obj) syncher.finish()
def import_departments(self, noop=False): obj_list = self.pk_get('department') syncher = ModelSyncher(Department.objects.all(), lambda obj: obj.id) self.dept_syncher = syncher if noop: return for d in obj_list: obj = syncher.get(d['id']) if not obj: obj = Department(id=d['id']) obj._changed = True self._save_translated_field(obj, 'name', d, 'name') if obj.abbr != d['abbr']: obj._changed = True obj.abbr = d['abbr'] if self.org_syncher: org_obj = self.org_syncher.get(d['org_id']) else: org_obj = Organization.objects.get(id=d['org_id']) assert org_obj if obj.organization_id != d['org_id']: obj._changed = True obj.organization = org_obj if obj._changed: obj.save() syncher.mark(obj) syncher.finish()
def import_services(self): obj_list = self.pk_get('service') syncher = ModelSyncher(Service.objects.all(), lambda obj: obj.id) for d in obj_list: obj = syncher.get(d['id']) if not obj: obj = Service(id=d['id']) obj._changed = True self._save_translated_field(obj, 'name', d, 'name') if 'parent_id' in d: parent = syncher.get(d['parent_id']) assert parent else: parent = None if obj.parent != parent: obj.parent = parent obj._changed = True if obj._changed: obj.save() syncher.mark(obj) syncher.finish()
def import_departments(noop=False, logger=None, fetch_resource=pk_get): obj_list = fetch_resource("department") syncher = ModelSyncher(Department.objects.all(), lambda obj: str(obj.uuid)) # self.dept_syncher = syncher if noop: return syncher for d in sorted(obj_list, key=lambda x: x["hierarchy_level"]): obj = syncher.get(d["id"]) obj_has_changed = False if not obj: obj = Department(uuid=d["id"]) obj_has_changed = True fields = ("phone", "address_zip", "oid", "organization_type", "business_id") fields_that_need_translation = ( "name", "abbr", "street_address", "address_city", "address_postal_full", "www", ) obj.uuid = d["id"] for field in fields: if d.get(field): if d[field] != getattr(obj, field): obj_has_changed = True setattr(obj, field, d.get(field)) parent_id = d.get("parent_id") if parent_id != obj.parent_id: obj_has_changed = True if parent_id is None: obj.parent_id = None else: try: parent = Department.objects.get(uuid=parent_id) obj.parent_id = parent.id except Department.DoesNotExist: logger and logger.error( "Department import: no parent with uuid {} found for {}" .format(parent_id, d["id"])) for field in fields_that_need_translation: if save_translated_field(obj, field, d, field): obj_has_changed = True muni_code = d.get("municipality_code") if muni_code is None: municipality = None if muni_code is not None: try: municipality = Municipality.objects.get( division__origin_id=str(muni_code)) except Municipality.DoesNotExist: logger and logger.error( "No municipality with code {} for department {}".format( muni_code, d["id"])) if obj.municipality != municipality: obj.municipality = municipality obj_has_changed = True if obj_has_changed: obj.save() syncher.mark(obj) syncher.finish() return syncher
def _import_unit_accessibility_properties(self): property_syncher = ModelSyncher( UnitAccessibilityProperty.objects.all(), lambda obj: obj.id) num_of_imports = 0 # For caching unit ids that are not present in the database unit_skip_list = set([]) accessibility_properties = get_ar_servicepoint_accessibility_resource( "properties") for accessibility_property in accessibility_properties: # Make sure that we have all the necessary property attributes ptv_id = accessibility_property.get("servicePointId") accessibility_variable_id = accessibility_property.get( "variableId") accessibility_variable_value = accessibility_property.get("value") if not (ptv_id and accessibility_variable_id and accessibility_variable_value): continue # No need to check further if the unit has already been marked as non-existing if ptv_id in unit_skip_list: continue # Make sure that the unit exists try: # TODO: Optimize this if it gets too slow # One way is to get all unit ids in one go and make a lookup table unit_identifier = UnitIdentifier.objects.get(namespace="ptv", value=ptv_id) unit = unit_identifier.unit except UnitIdentifier.DoesNotExist: self.logger.info( "Unit {} does not exist, skipping".format(ptv_id)) unit_skip_list.add(ptv_id) continue # Make sure that the variable exists if accessibility_variable_id not in self._accessibility_variables: self.logger.info("No variable {}, skipping".format(ptv_id)) continue # Create or update the property including its associated value uap, created = UnitAccessibilityProperty.objects.update_or_create( unit=unit, variable_id=accessibility_variable_id, defaults={"value": accessibility_variable_value}, ) # If an entry was updated if not created: # Mark it as synced sync_uap = property_syncher.get(uap.id) if sync_uap: property_syncher.mark(sync_uap) if created: num_of_imports += 1 property_syncher.finish() self.logger.info( "Imported {} accessibility properties.".format(num_of_imports))
def import_departments(noop=False, logger=None, fetch_resource=pk_get): obj_list = fetch_resource('department') syncher = ModelSyncher(Department.objects.all(), lambda obj: str(obj.uuid)) # self.dept_syncher = syncher if noop: return syncher for d in sorted(obj_list, key=lambda x: x['hierarchy_level']): obj = syncher.get(d['id']) obj_has_changed = False if not obj: obj = Department(uuid=d['id']) obj_has_changed = True fields = ('phone', 'address_zip', 'oid', 'organization_type', 'business_id') fields_that_need_translation = ('name', 'abbr', 'street_address', 'address_city', 'address_postal_full', 'www') obj.uuid = d['id'] for field in fields: if d.get(field): if d[field] != getattr(obj, field): obj_has_changed = True setattr(obj, field, d.get(field)) parent_id = d.get('parent_id') if parent_id != obj.parent_id: obj_has_changed = True if parent_id is None: obj.parent_id = None else: try: parent = Department.objects.get(uuid=parent_id) obj.parent_id = parent.id except Department.DoesNotExist: logger and logger.error( "Department import: no parent with uuid {} found for {}".format( parent_id, d['id']) ) for field in fields_that_need_translation: if save_translated_field(obj, field, d, field): obj_has_changed = True muni_code = d.get('municipality_code') if muni_code is None: municipality = None if muni_code is not None: try: municipality = Municipality.objects.get(division__origin_id=str(muni_code)) except Municipality.DoesNotExist: logger and logger.error( "No municipality with code {} for department {}".format( muni_code, d['id'])) if obj.municipality != municipality: obj.municipality = municipality obj_has_changed = True if obj_has_changed: obj.save() syncher.mark(obj) syncher.finish() return syncher