def test_taxlots_merge_without_losing_pairings(self): # Create 2 pairings and distribute them to the two -Views. property_factory = FakePropertyFactory(organization=self.org) property_state_factory = FakePropertyStateFactory( organization=self.org) property_1 = property_factory.get_property() state_1 = property_state_factory.get_property_state() property_view_1 = PropertyView.objects.create(property=property_1, cycle=self.cycle, state=state_1) property_2 = property_factory.get_property() state_2 = property_state_factory.get_property_state() property_view_2 = PropertyView.objects.create(property=property_2, cycle=self.cycle, state=state_2) TaxLotProperty(primary=True, cycle_id=self.cycle.id, property_view_id=property_view_1.id, taxlot_view_id=self.view_1.id).save() TaxLotProperty(primary=True, cycle_id=self.cycle.id, property_view_id=property_view_2.id, taxlot_view_id=self.view_2.id).save() # Merge the taxlots url = reverse('api:v2:taxlots-merge') + '?organization_id={}'.format( self.org.pk) post_params = json.dumps({ 'state_ids': [self.state_2.pk, self.state_1.pk] # priority given to state_1 }) self.client.post(url, post_params, content_type='application/json') # There should still be 2 TaxLotProperties self.assertEqual(TaxLotProperty.objects.count(), 2) taxlot_view = TaxLotView.objects.first() paired_propertyview_ids = list( TaxLotProperty.objects.filter( taxlot_view_id=taxlot_view.id).values_list('property_view_id', flat=True)) self.assertCountEqual(paired_propertyview_ids, [property_view_1.id, property_view_2.id])
def _copy_taxlotview_relationships(view_ids, new_view): """ Currently, TaxLotView relationships include notes, labels, and pairings with PropertyViews. Each of these are copied from given -View IDs and associated with a new TaxLotView. """ # Assign notes to the new view notes = list( Note.objects.values( 'name', 'note_type', 'text', 'log_data', 'created', 'updated', 'organization_id', 'user_id').filter(taxlot_view_id__in=view_ids).distinct()) for note in notes: note['taxlot_view'] = new_view n = Note(**note) n.save() # Correct the created and updated times to match the original note Note.objects.filter(id=n.id).update(created=note['created'], updated=note['updated']) # Associate labels TaxLotViewLabels = apps.get_model('seed', 'TaxLotView_labels') label_ids = list( TaxLotViewLabels.objects.filter( taxlotview_id__in=view_ids).values_list('statuslabel_id', flat=True)) new_view.labels.set(StatusLabel.objects.filter(pk__in=label_ids)) # Associate pairs while deleting old relationships paired_view_ids = list( TaxLotProperty.objects.filter(taxlot_view_id__in=view_ids).order_by( 'property_view_id').distinct('property_view_id').values_list( 'property_view_id', flat=True)) TaxLotProperty.objects.filter(taxlot_view_id__in=view_ids).delete() for paired_view_id in paired_view_ids: TaxLotProperty(primary=True, cycle_id=new_view.cycle_id, property_view_id=paired_view_id, taxlot_view_id=new_view.id).save()
def pair_unpair_property_taxlot(property_id, taxlot_id, organization_id, pair): # TODO: validate against organization_id, make sure cycle_ids are the same try: property_view = PropertyView.objects.get(pk=property_id) except PropertyView.DoesNotExist: return JsonResponse( { 'status': 'error', 'message': 'property view with id {} does not exist'.format(property_id) }, status=status.HTTP_404_NOT_FOUND) try: taxlot_view = TaxLotView.objects.get(pk=taxlot_id) except TaxLotView.DoesNotExist: return JsonResponse( { 'status': 'error', 'message': 'tax lot view with id {} does not exist'.format(taxlot_id) }, status=status.HTTP_404_NOT_FOUND) pv_cycle = property_view.cycle_id tv_cycle = taxlot_view.cycle_id if pv_cycle != tv_cycle: return JsonResponse( { 'status': 'error', 'message': 'Cycle mismatch between PropertyView and TaxLotView' }, status=status.HTTP_400_BAD_REQUEST) if pair: string = 'pair' if TaxLotProperty.objects.filter(property_view_id=property_id, taxlot_view_id=taxlot_id).exists(): return JsonResponse({ 'status': 'success', 'message': 'taxlot {} and property {} are already {}ed'.format( taxlot_id, property_id, string) }) TaxLotProperty(primary=True, cycle_id=pv_cycle, property_view_id=property_id, taxlot_view_id=taxlot_id) \ .save() success = True else: string = 'unpair' if not TaxLotProperty.objects.filter( property_view_id=property_id, taxlot_view_id=taxlot_id).exists(): return JsonResponse({ 'status': 'success', 'message': 'taxlot {} and property {} are already {}ed'.format( taxlot_id, property_id, string) }) TaxLotProperty.objects.filter(property_view_id=property_id, taxlot_view_id=taxlot_id) \ .delete() success = True if success: return JsonResponse({ 'status': 'success', 'message': 'taxlot {} and property {} are now {}ed'.format( taxlot_id, property_id, string) }) else: return JsonResponse( { 'status': 'error', 'message': 'Could not {} because reasons, maybe bad organization id={}'. format(string, organization_id) }, status=status.HTTP_500_INTERNAL_SERVER_ERROR)
def unmerge(self, request, pk=None): """ Unmerge a taxlot view into two taxlot views --- parameters: - name: organization_id description: The organization_id for this user's organization required: true paramType: query """ try: old_view = TaxLotView.objects.select_related( 'taxlot', 'cycle', 'state').get(id=pk, taxlot__organization_id=self. request.GET['organization_id']) except TaxLotView.DoesNotExist: return { 'status': 'error', 'message': 'taxlot view with id {} does not exist'.format(pk) } # Duplicate pairing paired_view_ids = list( TaxLotProperty.objects.filter(taxlot_view_id=old_view.id).order_by( 'property_view_id').values_list('property_view_id', flat=True)) # Capture previous associated labels label_ids = list(old_view.labels.all().values_list('id', flat=True)) notes = old_view.notes.all() for note in notes: note.taxlot_view = None merged_state = old_view.state if merged_state.data_state != DATA_STATE_MATCHING or merged_state.merge_state != MERGE_STATE_MERGED: return { 'status': 'error', 'message': 'taxlot view with id {} is not a merged taxlot view'.format(pk) } log = TaxLotAuditLog.objects.select_related( 'parent_state1', 'parent_state2').filter( state=merged_state).order_by('-id').first() if log.parent_state1 is None or log.parent_state2 is None: return { 'status': 'error', 'message': 'taxlot view with id {} must have two parent states'.format(pk) } state1 = log.parent_state1 state2 = log.parent_state2 cycle_id = old_view.cycle_id # Clone the taxlot record twice old_taxlot = old_view.taxlot new_taxlot = old_taxlot new_taxlot.id = None new_taxlot.save() new_taxlot_2 = TaxLot.objects.get(pk=new_taxlot.pk) new_taxlot_2.id = None new_taxlot_2.save() # If the canonical TaxLot is NOT associated to another -View if not TaxLotView.objects.filter(taxlot_id=old_view.taxlot_id).exclude( pk=old_view.id).exists(): TaxLot.objects.get(pk=old_view.taxlot_id).delete() # Create the views new_view1 = TaxLotView(cycle_id=cycle_id, taxlot_id=new_taxlot.id, state=state1) new_view2 = TaxLotView(cycle_id=cycle_id, taxlot_id=new_taxlot_2.id, state=state2) # Mark the merged state as deleted merged_state.merge_state = MERGE_STATE_DELETE merged_state.save() # Change the merge_state of the individual states if log.parent1.name in ['Import Creation', 'Manual Edit' ] and log.parent1.import_filename is not None: # State belongs to a new record state1.merge_state = MERGE_STATE_NEW else: state1.merge_state = MERGE_STATE_MERGED if log.parent2.name in ['Import Creation', 'Manual Edit' ] and log.parent2.import_filename is not None: # State belongs to a new record state2.merge_state = MERGE_STATE_NEW else: state2.merge_state = MERGE_STATE_MERGED # In most cases data_state will already be 3 (DATA_STATE_MATCHING), but if one of the parents was a # de-duplicated record then data_state will be 0. This step ensures that the new states will be 3. state1.data_state = DATA_STATE_MATCHING state2.data_state = DATA_STATE_MATCHING state1.save() state2.save() # Delete the audit log entry for the merge log.delete() old_view.delete() new_view1.save() new_view2.save() # Asssociate labels label_objs = StatusLabel.objects.filter(pk__in=label_ids) new_view1.labels.set(label_objs) new_view2.labels.set(label_objs) # Duplicate notes to the new views for note in notes: created = note.created updated = note.updated note.id = None note.taxlot_view = new_view1 note.save() ids = [note.id] note.id = None note.taxlot_view = new_view2 note.save() ids.append(note.id) # Correct the created and updated times to match the original note Note.objects.filter(id__in=ids).update(created=created, updated=updated) for paired_view_id in paired_view_ids: TaxLotProperty(primary=True, cycle_id=cycle_id, taxlot_view_id=new_view1.id, property_view_id=paired_view_id).save() TaxLotProperty(primary=True, cycle_id=cycle_id, taxlot_view_id=new_view2.id, property_view_id=paired_view_id).save() return {'status': 'success', 'view_id': new_view1.id}
def merge(self, request): """ Merge multiple tax lot records into a single new record --- parameters: - name: organization_id description: The organization_id for this user's organization required: true paramType: query - name: state_ids description: Array containing tax lot state ids to merge paramType: body """ body = request.data state_ids = body.get('state_ids', []) organization_id = int(request.query_params.get('organization_id', None)) # Check the number of state_ids to merge if len(state_ids) < 2: return JsonResponse( { 'status': 'error', 'message': 'At least two ids are necessary to merge' }, status=status.HTTP_400_BAD_REQUEST) # Make sure the state isn't already matched for state_id in state_ids: if ImportFileViewSet.has_coparent(state_id, 'properties'): return JsonResponse( { 'status': 'error', 'message': 'Source state [' + state_id + '] is already matched' }, status=status.HTTP_400_BAD_REQUEST) audit_log = TaxLotAuditLog inventory = TaxLot label = apps.get_model('seed', 'TaxLot_labels') state = TaxLotState view = TaxLotView index = 1 merged_state = None while index < len(state_ids): # state 1 is the base, state 2 is merged on top of state 1 # Use index 0 the first time through, merged_state from then on if index == 1: state1 = state.objects.get(id=state_ids[index - 1]) else: state1 = merged_state state2 = state.objects.get(id=state_ids[index]) merged_state = state.objects.create( organization_id=organization_id) merged_state = merging.merge_state(merged_state, state1, state2, merging.get_state_attrs( [state1, state2]), default=state2) state_1_audit_log = audit_log.objects.filter(state=state1).first() state_2_audit_log = audit_log.objects.filter(state=state2).first() audit_log.objects.create(organization=state1.organization, parent1=state_1_audit_log, parent2=state_2_audit_log, parent_state1=state1, parent_state2=state2, state=merged_state, name='Manual Match', description='Automatic Merge', import_filename=None, record_type=AUDIT_IMPORT) # Set the merged_state to merged merged_state.data_state = DATA_STATE_MATCHING merged_state.merge_state = MERGE_STATE_MERGED merged_state.save() state1.merge_state = MERGE_STATE_UNKNOWN state1.save() state2.merge_state = MERGE_STATE_UNKNOWN state2.save() # Delete existing views and inventory records views = view.objects.filter(state_id__in=[state1.id, state2.id]) view_ids = list(views.values_list('id', flat=True)) # Find unique notes notes = list( Note.objects.values( 'name', 'note_type', 'text', 'log_data', 'created', 'updated', 'organization_id', 'user_id').filter(taxlot_view_id__in=view_ids).distinct()) cycle_id = views.first().cycle_id label_ids = [] # Get paired view ids paired_view_ids = list( TaxLotProperty.objects.filter( taxlot_view_id__in=view_ids).order_by('property_view_id'). distinct('property_view_id').values_list('property_view_id', flat=True)) for v in views: label_ids.extend( list(v.taxlot.labels.all().values_list('id', flat=True))) v.taxlot.delete() label_ids = list(set(label_ids)) # Create new inventory record inventory_record = inventory(organization_id=organization_id) inventory_record.save() # Create new labels and view for label_id in label_ids: label(taxlot_id=inventory_record.id, statuslabel_id=label_id).save() new_view = view(cycle_id=cycle_id, state_id=merged_state.id, taxlot_id=inventory_record.id) new_view.save() # Assign notes to the new view for note in notes: note['taxlot_view'] = new_view n = Note(**note) n.save() # Correct the created and updated times to match the original note Note.objects.filter(id=n.id).update(created=note['created'], updated=note['updated']) # Delete existing pairs and re-pair all to new view # Probably already deleted by cascade TaxLotProperty.objects.filter(taxlot_view_id__in=view_ids).delete() for paired_view_id in paired_view_ids: TaxLotProperty(primary=True, cycle_id=cycle_id, property_view_id=paired_view_id, taxlot_view_id=new_view.id).save() index += 1 return {'status': 'success'}
def test_merged_indicators_provided_on_filter_endpoint(self): _import_record, import_file_1 = self.create_import_file( self.user, self.org, self.cycle) base_details = { 'address_line_1': '123 Match Street', 'import_file_id': import_file_1.id, 'data_state': DATA_STATE_MAPPING, 'no_default_data': False, } self.taxlot_state_factory.get_taxlot_state(**base_details) # set import_file_1 mapping done so that record is "created for users to view". import_file_1.mapping_done = True import_file_1.save() match_buildings(import_file_1.id) _import_record_2, import_file_2 = self.create_import_file( self.user, self.org, self.cycle) url = reverse( 'api:v2:taxlots-filter' ) + '?cycle_id={}&organization_id={}&page=1&per_page=999999999'.format( self.cycle.pk, self.org.pk) response = self.client.post(url) data = json.loads(response.content) self.assertFalse(data['results'][0]['merged_indicator']) # make sure merged_indicator is True when merge occurs base_details['city'] = 'Denver' base_details['import_file_id'] = import_file_2.id self.taxlot_state_factory.get_taxlot_state(**base_details) # set import_file_2 mapping done so that match merging can occur. import_file_2.mapping_done = True import_file_2.save() match_buildings(import_file_2.id) url = reverse( 'api:v2:taxlots-filter' ) + '?cycle_id={}&organization_id={}&page=1&per_page=999999999'.format( self.cycle.pk, self.org.pk) response = self.client.post(url) data = json.loads(response.content) self.assertTrue(data['results'][0]['merged_indicator']) # Create pairings and check if paired object has indicator as well property_factory = FakePropertyFactory(organization=self.org) property_state_factory = FakePropertyStateFactory( organization=self.org) property = property_factory.get_property() property_state = property_state_factory.get_property_state() property_view = PropertyView.objects.create(property=property, cycle=self.cycle, state=property_state) # attach pairing to one and only taxlot_view TaxLotProperty(primary=True, cycle_id=self.cycle.id, property_view_id=property_view.id, taxlot_view_id=TaxLotView.objects.get().id).save() url = reverse( 'api:v2:taxlots-filter' ) + '?cycle_id={}&organization_id={}&page=1&per_page=999999999'.format( self.cycle.pk, self.org.pk) response = self.client.post(url) data = json.loads(response.content) related = data['results'][0]['related'][0] self.assertTrue('merged_indicator' in related) self.assertFalse(related['merged_indicator'])
def unmerge(self, request, pk=None): """ Unmerge a taxlot view into two taxlot views --- parameters: - name: organization_id description: The organization_id for this user's organization required: true paramType: query """ try: old_view = TaxLotView.objects.select_related( 'taxlot', 'cycle', 'state').get(id=pk, taxlot__organization_id=self. request.GET['organization_id']) except TaxLotView.DoesNotExist: return { 'status': 'error', 'message': 'taxlot view with id {} does not exist'.format(pk) } notes = old_view.notes.all() for note in notes: note.taxlot_view = None merged_state = old_view.state if merged_state.data_state != DATA_STATE_MATCHING or merged_state.merge_state != MERGE_STATE_MERGED: return { 'status': 'error', 'message': 'taxlot view with id {} is not a merged taxlot view'.format(pk) } log = TaxLotAuditLog.objects.select_related( 'parent_state1', 'parent_state2').filter( state=merged_state).order_by('-id').first() if log.parent_state1 is None or log.parent_state2 is None: return { 'status': 'error', 'message': 'taxlot view with id {} must have two parent states'.format(pk) } label = apps.get_model('seed', 'TaxLot_labels') state1 = log.parent_state1 state2 = log.parent_state2 cycle_id = old_view.cycle_id # Clone the taxlot record, then the labels old_taxlot = old_view.taxlot label_ids = list(old_taxlot.labels.all().values_list('id', flat=True)) new_taxlot = old_taxlot new_taxlot.id = None new_taxlot.save() for label_id in label_ids: label(taxlot_id=new_taxlot.id, statuslabel_id=label_id).save() # Create the views new_view1 = TaxLotView(cycle_id=cycle_id, taxlot_id=new_taxlot.id, state=state1) new_view2 = TaxLotView(cycle_id=cycle_id, taxlot_id=old_view.taxlot_id, state=state2) # Mark the merged state as deleted merged_state.merge_state = MERGE_STATE_DELETE merged_state.save() # Change the merge_state of the individual states if log.parent1.name in ['Import Creation', 'Manual Edit' ] and log.parent1.import_filename is not None: # State belongs to a new record state1.merge_state = MERGE_STATE_NEW else: state1.merge_state = MERGE_STATE_MERGED if log.parent2.name in ['Import Creation', 'Manual Edit' ] and log.parent2.import_filename is not None: # State belongs to a new record state2.merge_state = MERGE_STATE_NEW else: state2.merge_state = MERGE_STATE_MERGED state1.save() state2.save() # Delete the audit log entry for the merge log.delete() # Duplicate pairing paired_view_ids = list( TaxLotProperty.objects.filter(taxlot_view_id=old_view.id).order_by( 'property_view_id').values_list('property_view_id', flat=True)) old_view.delete() new_view1.save() new_view2.save() # Duplicate notes to the new views for note in notes: created = note.created updated = note.updated note.id = None note.taxlot_view = new_view1 note.save() ids = [note.id] note.id = None note.taxlot_view = new_view2 note.save() ids.append(note.id) # Correct the created and updated times to match the original note Note.objects.filter(id__in=ids).update(created=created, updated=updated) for paired_view_id in paired_view_ids: TaxLotProperty(primary=True, cycle_id=cycle_id, taxlot_view_id=new_view1.id, property_view_id=paired_view_id).save() TaxLotProperty(primary=True, cycle_id=cycle_id, taxlot_view_id=new_view2.id, property_view_id=paired_view_id).save() return {'status': 'success', 'view_id': new_view1.id}