def test_taxlots_merge_without_losing_pairings(self):
# Create 2 pairings and distribute them to the two -Views.
property_factory = FakePropertyFactory(organization=self.org)
property_state_factory = FakePropertyStateFactory(
organization=self.org)
property_1 = property_factory.get_property()
state_1 = property_state_factory.get_property_state()
property_view_1 = PropertyView.objects.create(property=property_1,
cycle=self.cycle,
state=state_1)
property_2 = property_factory.get_property()
state_2 = property_state_factory.get_property_state()
property_view_2 = PropertyView.objects.create(property=property_2,
cycle=self.cycle,
state=state_2)
TaxLotProperty(primary=True,
cycle_id=self.cycle.id,
property_view_id=property_view_1.id,
taxlot_view_id=self.view_1.id).save()
TaxLotProperty(primary=True,
cycle_id=self.cycle.id,
property_view_id=property_view_2.id,
taxlot_view_id=self.view_2.id).save()
# Merge the taxlots
url = reverse('api:v2:taxlots-merge') + '?organization_id={}'.format(
self.org.pk)
post_params = json.dumps({
'state_ids': [self.state_2.pk,
self.state_1.pk] # priority given to state_1
})
self.client.post(url, post_params, content_type='application/json')
# There should still be 2 TaxLotProperties
self.assertEqual(TaxLotProperty.objects.count(), 2)
taxlot_view = TaxLotView.objects.first()
paired_propertyview_ids = list(
TaxLotProperty.objects.filter(
taxlot_view_id=taxlot_view.id).values_list('property_view_id',
flat=True))
self.assertCountEqual(paired_propertyview_ids,
[property_view_1.id, property_view_2.id])
def _copy_taxlotview_relationships(view_ids, new_view):
"""
Currently, TaxLotView relationships include notes, labels, and pairings
with PropertyViews. Each of these are copied from given -View IDs and
associated with a new TaxLotView.
"""
# Assign notes to the new view
notes = list(
Note.objects.values(
'name', 'note_type', 'text', 'log_data', 'created', 'updated',
'organization_id',
'user_id').filter(taxlot_view_id__in=view_ids).distinct())
for note in notes:
note['taxlot_view'] = new_view
n = Note(**note)
n.save()
# Correct the created and updated times to match the original note
Note.objects.filter(id=n.id).update(created=note['created'],
updated=note['updated'])
# Associate labels
TaxLotViewLabels = apps.get_model('seed', 'TaxLotView_labels')
label_ids = list(
TaxLotViewLabels.objects.filter(
taxlotview_id__in=view_ids).values_list('statuslabel_id',
flat=True))
new_view.labels.set(StatusLabel.objects.filter(pk__in=label_ids))
# Associate pairs while deleting old relationships
paired_view_ids = list(
TaxLotProperty.objects.filter(taxlot_view_id__in=view_ids).order_by(
'property_view_id').distinct('property_view_id').values_list(
'property_view_id', flat=True))
TaxLotProperty.objects.filter(taxlot_view_id__in=view_ids).delete()
for paired_view_id in paired_view_ids:
TaxLotProperty(primary=True,
cycle_id=new_view.cycle_id,
property_view_id=paired_view_id,
taxlot_view_id=new_view.id).save()
def pair_unpair_property_taxlot(property_id, taxlot_id, organization_id, pair):
# TODO: validate against organization_id, make sure cycle_ids are the same
try:
property_view = PropertyView.objects.get(pk=property_id)
except PropertyView.DoesNotExist:
return JsonResponse(
{
'status':
'error',
'message':
'property view with id {} does not exist'.format(property_id)
},
status=status.HTTP_404_NOT_FOUND)
try:
taxlot_view = TaxLotView.objects.get(pk=taxlot_id)
except TaxLotView.DoesNotExist:
return JsonResponse(
{
'status':
'error',
'message':
'tax lot view with id {} does not exist'.format(taxlot_id)
},
status=status.HTTP_404_NOT_FOUND)
pv_cycle = property_view.cycle_id
tv_cycle = taxlot_view.cycle_id
if pv_cycle != tv_cycle:
return JsonResponse(
{
'status': 'error',
'message': 'Cycle mismatch between PropertyView and TaxLotView'
},
status=status.HTTP_400_BAD_REQUEST)
if pair:
string = 'pair'
if TaxLotProperty.objects.filter(property_view_id=property_id,
taxlot_view_id=taxlot_id).exists():
return JsonResponse({
'status':
'success',
'message':
'taxlot {} and property {} are already {}ed'.format(
taxlot_id, property_id, string)
})
TaxLotProperty(primary=True, cycle_id=pv_cycle, property_view_id=property_id,
taxlot_view_id=taxlot_id) \
.save()
success = True
else:
string = 'unpair'
if not TaxLotProperty.objects.filter(
property_view_id=property_id,
taxlot_view_id=taxlot_id).exists():
return JsonResponse({
'status':
'success',
'message':
'taxlot {} and property {} are already {}ed'.format(
taxlot_id, property_id, string)
})
TaxLotProperty.objects.filter(property_view_id=property_id, taxlot_view_id=taxlot_id) \
.delete()
success = True
if success:
return JsonResponse({
'status':
'success',
'message':
'taxlot {} and property {} are now {}ed'.format(
taxlot_id, property_id, string)
})
else:
return JsonResponse(
{
'status':
'error',
'message':
'Could not {} because reasons, maybe bad organization id={}'.
format(string, organization_id)
},
status=status.HTTP_500_INTERNAL_SERVER_ERROR)
def unmerge(self, request, pk=None):
"""
Unmerge a taxlot view into two taxlot views
---
parameters:
- name: organization_id
description: The organization_id for this user's organization
required: true
paramType: query
"""
try:
old_view = TaxLotView.objects.select_related(
'taxlot', 'cycle', 'state').get(id=pk,
taxlot__organization_id=self.
request.GET['organization_id'])
except TaxLotView.DoesNotExist:
return {
'status': 'error',
'message': 'taxlot view with id {} does not exist'.format(pk)
}
# Duplicate pairing
paired_view_ids = list(
TaxLotProperty.objects.filter(taxlot_view_id=old_view.id).order_by(
'property_view_id').values_list('property_view_id', flat=True))
# Capture previous associated labels
label_ids = list(old_view.labels.all().values_list('id', flat=True))
notes = old_view.notes.all()
for note in notes:
note.taxlot_view = None
merged_state = old_view.state
if merged_state.data_state != DATA_STATE_MATCHING or merged_state.merge_state != MERGE_STATE_MERGED:
return {
'status':
'error',
'message':
'taxlot view with id {} is not a merged taxlot view'.format(pk)
}
log = TaxLotAuditLog.objects.select_related(
'parent_state1', 'parent_state2').filter(
state=merged_state).order_by('-id').first()
if log.parent_state1 is None or log.parent_state2 is None:
return {
'status':
'error',
'message':
'taxlot view with id {} must have two parent states'.format(pk)
}
state1 = log.parent_state1
state2 = log.parent_state2
cycle_id = old_view.cycle_id
# Clone the taxlot record twice
old_taxlot = old_view.taxlot
new_taxlot = old_taxlot
new_taxlot.id = None
new_taxlot.save()
new_taxlot_2 = TaxLot.objects.get(pk=new_taxlot.pk)
new_taxlot_2.id = None
new_taxlot_2.save()
# If the canonical TaxLot is NOT associated to another -View
if not TaxLotView.objects.filter(taxlot_id=old_view.taxlot_id).exclude(
pk=old_view.id).exists():
TaxLot.objects.get(pk=old_view.taxlot_id).delete()
# Create the views
new_view1 = TaxLotView(cycle_id=cycle_id,
taxlot_id=new_taxlot.id,
state=state1)
new_view2 = TaxLotView(cycle_id=cycle_id,
taxlot_id=new_taxlot_2.id,
state=state2)
# Mark the merged state as deleted
merged_state.merge_state = MERGE_STATE_DELETE
merged_state.save()
# Change the merge_state of the individual states
if log.parent1.name in ['Import Creation', 'Manual Edit'
] and log.parent1.import_filename is not None:
# State belongs to a new record
state1.merge_state = MERGE_STATE_NEW
else:
state1.merge_state = MERGE_STATE_MERGED
if log.parent2.name in ['Import Creation', 'Manual Edit'
] and log.parent2.import_filename is not None:
# State belongs to a new record
state2.merge_state = MERGE_STATE_NEW
else:
state2.merge_state = MERGE_STATE_MERGED
# In most cases data_state will already be 3 (DATA_STATE_MATCHING), but if one of the parents was a
# de-duplicated record then data_state will be 0. This step ensures that the new states will be 3.
state1.data_state = DATA_STATE_MATCHING
state2.data_state = DATA_STATE_MATCHING
state1.save()
state2.save()
# Delete the audit log entry for the merge
log.delete()
old_view.delete()
new_view1.save()
new_view2.save()
# Asssociate labels
label_objs = StatusLabel.objects.filter(pk__in=label_ids)
new_view1.labels.set(label_objs)
new_view2.labels.set(label_objs)
# Duplicate notes to the new views
for note in notes:
created = note.created
updated = note.updated
note.id = None
note.taxlot_view = new_view1
note.save()
ids = [note.id]
note.id = None
note.taxlot_view = new_view2
note.save()
ids.append(note.id)
# Correct the created and updated times to match the original note
Note.objects.filter(id__in=ids).update(created=created,
updated=updated)
for paired_view_id in paired_view_ids:
TaxLotProperty(primary=True,
cycle_id=cycle_id,
taxlot_view_id=new_view1.id,
property_view_id=paired_view_id).save()
TaxLotProperty(primary=True,
cycle_id=cycle_id,
taxlot_view_id=new_view2.id,
property_view_id=paired_view_id).save()
return {'status': 'success', 'view_id': new_view1.id}
def merge(self, request):
"""
Merge multiple tax lot records into a single new record
---
parameters:
- name: organization_id
description: The organization_id for this user's organization
required: true
paramType: query
- name: state_ids
description: Array containing tax lot state ids to merge
paramType: body
"""
body = request.data
state_ids = body.get('state_ids', [])
organization_id = int(request.query_params.get('organization_id',
None))
# Check the number of state_ids to merge
if len(state_ids) < 2:
return JsonResponse(
{
'status': 'error',
'message': 'At least two ids are necessary to merge'
},
status=status.HTTP_400_BAD_REQUEST)
# Make sure the state isn't already matched
for state_id in state_ids:
if ImportFileViewSet.has_coparent(state_id, 'properties'):
return JsonResponse(
{
'status':
'error',
'message':
'Source state [' + state_id + '] is already matched'
},
status=status.HTTP_400_BAD_REQUEST)
audit_log = TaxLotAuditLog
inventory = TaxLot
label = apps.get_model('seed', 'TaxLot_labels')
state = TaxLotState
view = TaxLotView
index = 1
merged_state = None
while index < len(state_ids):
# state 1 is the base, state 2 is merged on top of state 1
# Use index 0 the first time through, merged_state from then on
if index == 1:
state1 = state.objects.get(id=state_ids[index - 1])
else:
state1 = merged_state
state2 = state.objects.get(id=state_ids[index])
merged_state = state.objects.create(
organization_id=organization_id)
merged_state = merging.merge_state(merged_state,
state1,
state2,
merging.get_state_attrs(
[state1, state2]),
default=state2)
state_1_audit_log = audit_log.objects.filter(state=state1).first()
state_2_audit_log = audit_log.objects.filter(state=state2).first()
audit_log.objects.create(organization=state1.organization,
parent1=state_1_audit_log,
parent2=state_2_audit_log,
parent_state1=state1,
parent_state2=state2,
state=merged_state,
name='Manual Match',
description='Automatic Merge',
import_filename=None,
record_type=AUDIT_IMPORT)
# Set the merged_state to merged
merged_state.data_state = DATA_STATE_MATCHING
merged_state.merge_state = MERGE_STATE_MERGED
merged_state.save()
state1.merge_state = MERGE_STATE_UNKNOWN
state1.save()
state2.merge_state = MERGE_STATE_UNKNOWN
state2.save()
# Delete existing views and inventory records
views = view.objects.filter(state_id__in=[state1.id, state2.id])
view_ids = list(views.values_list('id', flat=True))
# Find unique notes
notes = list(
Note.objects.values(
'name', 'note_type', 'text', 'log_data', 'created',
'updated', 'organization_id',
'user_id').filter(taxlot_view_id__in=view_ids).distinct())
cycle_id = views.first().cycle_id
label_ids = []
# Get paired view ids
paired_view_ids = list(
TaxLotProperty.objects.filter(
taxlot_view_id__in=view_ids).order_by('property_view_id').
distinct('property_view_id').values_list('property_view_id',
flat=True))
for v in views:
label_ids.extend(
list(v.taxlot.labels.all().values_list('id', flat=True)))
v.taxlot.delete()
label_ids = list(set(label_ids))
# Create new inventory record
inventory_record = inventory(organization_id=organization_id)
inventory_record.save()
# Create new labels and view
for label_id in label_ids:
label(taxlot_id=inventory_record.id,
statuslabel_id=label_id).save()
new_view = view(cycle_id=cycle_id,
state_id=merged_state.id,
taxlot_id=inventory_record.id)
new_view.save()
# Assign notes to the new view
for note in notes:
note['taxlot_view'] = new_view
n = Note(**note)
n.save()
# Correct the created and updated times to match the original note
Note.objects.filter(id=n.id).update(created=note['created'],
updated=note['updated'])
# Delete existing pairs and re-pair all to new view
# Probably already deleted by cascade
TaxLotProperty.objects.filter(taxlot_view_id__in=view_ids).delete()
for paired_view_id in paired_view_ids:
TaxLotProperty(primary=True,
cycle_id=cycle_id,
property_view_id=paired_view_id,
taxlot_view_id=new_view.id).save()
index += 1
return {'status': 'success'}
def test_merged_indicators_provided_on_filter_endpoint(self):
_import_record, import_file_1 = self.create_import_file(
self.user, self.org, self.cycle)
base_details = {
'address_line_1': '123 Match Street',
'import_file_id': import_file_1.id,
'data_state': DATA_STATE_MAPPING,
'no_default_data': False,
}
self.taxlot_state_factory.get_taxlot_state(**base_details)
# set import_file_1 mapping done so that record is "created for users to view".
import_file_1.mapping_done = True
import_file_1.save()
match_buildings(import_file_1.id)
_import_record_2, import_file_2 = self.create_import_file(
self.user, self.org, self.cycle)
url = reverse(
'api:v2:taxlots-filter'
) + '?cycle_id={}&organization_id={}&page=1&per_page=999999999'.format(
self.cycle.pk, self.org.pk)
response = self.client.post(url)
data = json.loads(response.content)
self.assertFalse(data['results'][0]['merged_indicator'])
# make sure merged_indicator is True when merge occurs
base_details['city'] = 'Denver'
base_details['import_file_id'] = import_file_2.id
self.taxlot_state_factory.get_taxlot_state(**base_details)
# set import_file_2 mapping done so that match merging can occur.
import_file_2.mapping_done = True
import_file_2.save()
match_buildings(import_file_2.id)
url = reverse(
'api:v2:taxlots-filter'
) + '?cycle_id={}&organization_id={}&page=1&per_page=999999999'.format(
self.cycle.pk, self.org.pk)
response = self.client.post(url)
data = json.loads(response.content)
self.assertTrue(data['results'][0]['merged_indicator'])
# Create pairings and check if paired object has indicator as well
property_factory = FakePropertyFactory(organization=self.org)
property_state_factory = FakePropertyStateFactory(
organization=self.org)
property = property_factory.get_property()
property_state = property_state_factory.get_property_state()
property_view = PropertyView.objects.create(property=property,
cycle=self.cycle,
state=property_state)
# attach pairing to one and only taxlot_view
TaxLotProperty(primary=True,
cycle_id=self.cycle.id,
property_view_id=property_view.id,
taxlot_view_id=TaxLotView.objects.get().id).save()
url = reverse(
'api:v2:taxlots-filter'
) + '?cycle_id={}&organization_id={}&page=1&per_page=999999999'.format(
self.cycle.pk, self.org.pk)
response = self.client.post(url)
data = json.loads(response.content)
related = data['results'][0]['related'][0]
self.assertTrue('merged_indicator' in related)
self.assertFalse(related['merged_indicator'])
def unmerge(self, request, pk=None):
"""
Unmerge a taxlot view into two taxlot views
---
parameters:
- name: organization_id
description: The organization_id for this user's organization
required: true
paramType: query
"""
try:
old_view = TaxLotView.objects.select_related(
'taxlot', 'cycle', 'state').get(id=pk,
taxlot__organization_id=self.
request.GET['organization_id'])
except TaxLotView.DoesNotExist:
return {
'status': 'error',
'message': 'taxlot view with id {} does not exist'.format(pk)
}
notes = old_view.notes.all()
for note in notes:
note.taxlot_view = None
merged_state = old_view.state
if merged_state.data_state != DATA_STATE_MATCHING or merged_state.merge_state != MERGE_STATE_MERGED:
return {
'status':
'error',
'message':
'taxlot view with id {} is not a merged taxlot view'.format(pk)
}
log = TaxLotAuditLog.objects.select_related(
'parent_state1', 'parent_state2').filter(
state=merged_state).order_by('-id').first()
if log.parent_state1 is None or log.parent_state2 is None:
return {
'status':
'error',
'message':
'taxlot view with id {} must have two parent states'.format(pk)
}
label = apps.get_model('seed', 'TaxLot_labels')
state1 = log.parent_state1
state2 = log.parent_state2
cycle_id = old_view.cycle_id
# Clone the taxlot record, then the labels
old_taxlot = old_view.taxlot
label_ids = list(old_taxlot.labels.all().values_list('id', flat=True))
new_taxlot = old_taxlot
new_taxlot.id = None
new_taxlot.save()
for label_id in label_ids:
label(taxlot_id=new_taxlot.id, statuslabel_id=label_id).save()
# Create the views
new_view1 = TaxLotView(cycle_id=cycle_id,
taxlot_id=new_taxlot.id,
state=state1)
new_view2 = TaxLotView(cycle_id=cycle_id,
taxlot_id=old_view.taxlot_id,
state=state2)
# Mark the merged state as deleted
merged_state.merge_state = MERGE_STATE_DELETE
merged_state.save()
# Change the merge_state of the individual states
if log.parent1.name in ['Import Creation', 'Manual Edit'
] and log.parent1.import_filename is not None:
# State belongs to a new record
state1.merge_state = MERGE_STATE_NEW
else:
state1.merge_state = MERGE_STATE_MERGED
if log.parent2.name in ['Import Creation', 'Manual Edit'
] and log.parent2.import_filename is not None:
# State belongs to a new record
state2.merge_state = MERGE_STATE_NEW
else:
state2.merge_state = MERGE_STATE_MERGED
state1.save()
state2.save()
# Delete the audit log entry for the merge
log.delete()
# Duplicate pairing
paired_view_ids = list(
TaxLotProperty.objects.filter(taxlot_view_id=old_view.id).order_by(
'property_view_id').values_list('property_view_id', flat=True))
old_view.delete()
new_view1.save()
new_view2.save()
# Duplicate notes to the new views
for note in notes:
created = note.created
updated = note.updated
note.id = None
note.taxlot_view = new_view1
note.save()
ids = [note.id]
note.id = None
note.taxlot_view = new_view2
note.save()
ids.append(note.id)
# Correct the created and updated times to match the original note
Note.objects.filter(id__in=ids).update(created=created,
updated=updated)
for paired_view_id in paired_view_ids:
TaxLotProperty(primary=True,
cycle_id=cycle_id,
taxlot_view_id=new_view1.id,
property_view_id=paired_view_id).save()
TaxLotProperty(primary=True,
cycle_id=cycle_id,
taxlot_view_id=new_view2.id,
property_view_id=paired_view_id).save()
return {'status': 'success', 'view_id': new_view1.id}