def merge_taxlots(state_ids, org_id, log_name, ignore_merge_protection=False):
index = 1
merged_state = None
while index < len(state_ids):
# state 1 is the base, state 2 is merged on top of state 1
# Use index 0 the first time through, merged_state from then on
if index == 1:
state_1 = TaxLotState.objects.get(id=state_ids[index - 1])
else:
state_1 = merged_state
state_2 = TaxLotState.objects.get(id=state_ids[index])
merged_state = _merge_log_states(org_id, state_1, state_2, log_name,
ignore_merge_protection)
views = TaxLotView.objects.filter(
state_id__in=[state_1.id, state_2.id])
view_ids = list(views.values_list('id', flat=True))
canonical_ids = list(views.values_list('taxlot_id', flat=True))
# Create new inventory record and associate it to a new view
new_taxlot = TaxLot(organization_id=org_id)
new_taxlot.save()
cycle_id = views.first().cycle_id
new_view = TaxLotView(cycle_id=cycle_id,
state_id=merged_state.id,
taxlot_id=new_taxlot.id)
new_view.save()
_copy_taxlotview_relationships(view_ids, new_view)
# Delete canonical records that are NOT associated to other -Views.
other_associated_views = TaxLotView.objects.filter(
taxlot_id__in=canonical_ids).exclude(pk__in=view_ids)
TaxLot.objects \
.filter(pk__in=canonical_ids) \
.exclude(pk__in=Subquery(other_associated_views.values('taxlot_id'))) \
.delete()
# Delete all -Views
TaxLotView.objects.filter(pk__in=view_ids).delete()
index += 1
return merged_state
def unmerge(self, request, pk=None):
"""
Unmerge a taxlot view into two taxlot views
---
parameters:
- name: organization_id
description: The organization_id for this user's organization
required: true
paramType: query
"""
try:
old_view = TaxLotView.objects.select_related(
'taxlot', 'cycle', 'state').get(id=pk,
taxlot__organization_id=self.
request.GET['organization_id'])
except TaxLotView.DoesNotExist:
return {
'status': 'error',
'message': 'taxlot view with id {} does not exist'.format(pk)
}
# Duplicate pairing
paired_view_ids = list(
TaxLotProperty.objects.filter(taxlot_view_id=old_view.id).order_by(
'property_view_id').values_list('property_view_id', flat=True))
# Capture previous associated labels
label_ids = list(old_view.labels.all().values_list('id', flat=True))
notes = old_view.notes.all()
for note in notes:
note.taxlot_view = None
merged_state = old_view.state
if merged_state.data_state != DATA_STATE_MATCHING or merged_state.merge_state != MERGE_STATE_MERGED:
return {
'status':
'error',
'message':
'taxlot view with id {} is not a merged taxlot view'.format(pk)
}
log = TaxLotAuditLog.objects.select_related(
'parent_state1', 'parent_state2').filter(
state=merged_state).order_by('-id').first()
if log.parent_state1 is None or log.parent_state2 is None:
return {
'status':
'error',
'message':
'taxlot view with id {} must have two parent states'.format(pk)
}
state1 = log.parent_state1
state2 = log.parent_state2
cycle_id = old_view.cycle_id
# Clone the taxlot record twice
old_taxlot = old_view.taxlot
new_taxlot = old_taxlot
new_taxlot.id = None
new_taxlot.save()
new_taxlot_2 = TaxLot.objects.get(pk=new_taxlot.pk)
new_taxlot_2.id = None
new_taxlot_2.save()
# If the canonical TaxLot is NOT associated to another -View
if not TaxLotView.objects.filter(taxlot_id=old_view.taxlot_id).exclude(
pk=old_view.id).exists():
TaxLot.objects.get(pk=old_view.taxlot_id).delete()
# Create the views
new_view1 = TaxLotView(cycle_id=cycle_id,
taxlot_id=new_taxlot.id,
state=state1)
new_view2 = TaxLotView(cycle_id=cycle_id,
taxlot_id=new_taxlot_2.id,
state=state2)
# Mark the merged state as deleted
merged_state.merge_state = MERGE_STATE_DELETE
merged_state.save()
# Change the merge_state of the individual states
if log.parent1.name in ['Import Creation', 'Manual Edit'
] and log.parent1.import_filename is not None:
# State belongs to a new record
state1.merge_state = MERGE_STATE_NEW
else:
state1.merge_state = MERGE_STATE_MERGED
if log.parent2.name in ['Import Creation', 'Manual Edit'
] and log.parent2.import_filename is not None:
# State belongs to a new record
state2.merge_state = MERGE_STATE_NEW
else:
state2.merge_state = MERGE_STATE_MERGED
# In most cases data_state will already be 3 (DATA_STATE_MATCHING), but if one of the parents was a
# de-duplicated record then data_state will be 0. This step ensures that the new states will be 3.
state1.data_state = DATA_STATE_MATCHING
state2.data_state = DATA_STATE_MATCHING
state1.save()
state2.save()
# Delete the audit log entry for the merge
log.delete()
old_view.delete()
new_view1.save()
new_view2.save()
# Asssociate labels
label_objs = StatusLabel.objects.filter(pk__in=label_ids)
new_view1.labels.set(label_objs)
new_view2.labels.set(label_objs)
# Duplicate notes to the new views
for note in notes:
created = note.created
updated = note.updated
note.id = None
note.taxlot_view = new_view1
note.save()
ids = [note.id]
note.id = None
note.taxlot_view = new_view2
note.save()
ids.append(note.id)
# Correct the created and updated times to match the original note
Note.objects.filter(id__in=ids).update(created=created,
updated=updated)
for paired_view_id in paired_view_ids:
TaxLotProperty(primary=True,
cycle_id=cycle_id,
taxlot_view_id=new_view1.id,
property_view_id=paired_view_id).save()
TaxLotProperty(primary=True,
cycle_id=cycle_id,
taxlot_view_id=new_view2.id,
property_view_id=paired_view_id).save()
return {'status': 'success', 'view_id': new_view1.id}
def split_taxlots_into_m2m_relationships(self, org_id, org_rules_map):
org = Organization.objects.get(pk=org_id)
logging_info("Splitting tax lot lists for organization {}/{}".format(
org_id, org.name))
created_tax_lots = collections.defaultdict(lambda: False)
for m2m in itertools.chain(
TaxLotProperty.objects.filter(
property_view__property__organization=org).all(),
TaxLotProperty.objects.filter(
taxlot_view__taxlot__organization=org).all()):
# aggregate_value_from_state(view.state, org_rules_map[org_id])
# In some cases something in this chain of db calls in m2m.taxlot_view.state.jurisdiction_tax_lot_id
# something is missing. Log it and continue.
try:
jurisdiction_tax_lot_id = m2m.taxlot_view.state.jurisdiction_tax_lot_id
except Exception as e:
logging_error(
"Error splitting taxlotproperty {t} into m2m: {e}".format(
t=m2m, e=e))
continue
logging_info(
"Starting to do m2m for jurisdiction_tax_lot_id {id}".format(
id=jurisdiction_tax_lot_id))
taxlot_id_list = []
try:
taxlot_id_list = get_id_fields(
m2m.taxlot_view.state.jurisdiction_tax_lot_id)
_log.info("Found taxlot_id_list: {l}".format(l=taxlot_id_list))
except TaxLotIDValueError, e:
logging_warn(e)
continue
if len(taxlot_id_list) <= 1: continue
logging_info(
"Tax lot view {} w/ tax_lot id {} was split to {} elements: {}"
.format(m2m.taxlot_view.pk,
m2m.taxlot_view.state.jurisdiction_tax_lot_id,
len(taxlot_id_list), taxlot_id_list))
original_taxlot_view = m2m.taxlot_view
# Some have duplicates
for tax_lot_id in set(taxlot_id_list):
logging_info("Break up tax lot {} to {} for cycle {}".format(
tax_lot_id, taxlot_id_list, m2m.cycle))
# Take tax lot and create a taxlot, a taxlot view, and a taxlot state.
# taxlot state, and an m2m for the view and installs each.
# Check to see if the tax lot exists
matching_views_qry = TaxLotView.objects.filter(
taxlot__organization=org,
state__jurisdiction_tax_lot_id=tax_lot_id)
matching_views_ct = matching_views_qry.count()
logging_info(
"Found {ct} matching views".format(ct=matching_views_ct))
if matching_views_qry.count():
tax_lot = matching_views_qry.first().taxlot
state = matching_views_qry.first().state
logging_info(
"Found matching taxlotviews. First is jurisdiction_tax_lot_id {id}"
.format(id=state.jurisdiction_tax_lot_id))
# FIXME: Yuck! Refactor me please!
created_tax_lots[tax_lot_id] = tax_lot
logging_info(
"Setting taxlot_state to jurisdiction_tax_lot_id {id}".
format(id=original_taxlot_view.state.
jurisdiction_tax_lot_id))
# Apparently this is how Django clones things?
taxlot_state = original_taxlot_view.state
taxlot_state.pk = None
taxlot_state.jurisdiction_tax_lot_id = tax_lot_id
logging_info(
"Setting taxlot_state.jurisdiction_tax_lot_id = {id}".
format(id=tax_lot_id))
taxlot_state.save()
else:
logging_info("No match, make a new TaxLot")
tl = TaxLot(
organization=m2m.taxlot_view.taxlot.organization)
tl.save()
created_tax_lots[tax_lot_id] = tl
logging_info(
"Setting taxlot_state to jurisdiction_tax_lot_id {id}".
format(id=original_taxlot_view.state.
jurisdiction_tax_lot_id))
# Apparently this is how Django clones things?
taxlot_state = original_taxlot_view.state
taxlot_state.pk = None
taxlot_state.jurisdiction_tax_lot_id = tax_lot_id
logging_info(
"Setting taxlot_state.jurisdiction_tax_lot_id = {id}".
format(id=tax_lot_id))
taxlot_state.save()
# Check and see if the Tax Lot View exists
qry = TaxLotView.objects.filter(
taxlot=created_tax_lots[tax_lot_id], cycle=m2m.cycle)
taxlotview_ct = qry.count()
logging_info(
"Found {ct} matching taxlotviews".format(ct=taxlotview_ct))
if taxlotview_ct:
taxlotview = qry.first()
logging_debug("Setting the state of {v} to {s}".format(
v=taxlotview.state.jurisdiction_tax_lot_id,
s=taxlot_state.jurisdiction_tax_lot_id))
taxlotview.state = taxlot_state
taxlotview.save()
else:
logging_debug(
"Creating a new TaxLotView with cycle {c} and state {s}"
.format(c=m2m.cycle.name,
s=taxlot_state.jurisdiction_tax_lot_id))
taxlotview = TaxLotView(
taxlot=created_tax_lots[tax_lot_id],
cycle=m2m.cycle,
state=taxlot_state)
# Clone the state from above
taxlotview.save()
logging_debug(
"TaxLotProperty.objects.get_or_create with pm_id {pm}, jurisdiction_id = {j}, cycle = {c}"
.format(pm=m2m.property_view.state.pm_property_id,
j=taxlotview.state.jurisdiction_tax_lot_id,
c=m2m.cycle.name))
TaxLotProperty.objects.get_or_create(
property_view=m2m.property_view,
taxlot_view=taxlotview,
cycle=m2m.cycle)
else:
# The existing TaxLotView and m2m is deleted.
logging_debug(
"Deleting existing TaxLotView pm {pm}, jurisdiction {j}".
format(pm=m2m.property_view.state.pm_property_id,
j=m2m.taxlot_view.state.jurisdiction_tax_lot_id))
tl_view = m2m.taxlot_view
m2m.delete()
tl_view.delete()
pass
# Go through each view, find all it's tax lot ids and make sure they don't look like lists of many things.
logging_info("{} => {}".format(jurisdiction_tax_lot_id,
taxlot_id_list))
taxlotview_ct = TaxLotView.objects.filter(
taxlot=taxlot,
cycle=original_taxlot_view.cycle).count()
logging_debug(
"Found {ct} taxlotviews".format(ct=taxlotview_ct))
if taxlotview_ct == 0:
taxlot_state = original_taxlot_view.state
taxlot_state.pk = None
taxlot_state.jurisdiction_tax_lot_id = taxlot_id
logging_debug(
"Creating a copy of the original taxlot_view's state with jurisdiction id {j}"
.format(j=taxlot_id))
taxlot_state.save()
logging_debug("Creating a new TaxLotView")
tlv = TaxLotView(taxlot=taxlot,
cycle=original_taxlot_view.cycle,
state=taxlot_state)
tlv.save()
else:
logging_debug("Creating a new TaxLot")
tl = TaxLot(organization=original_taxlot_view.taxlot.
organization)
tl.save()
logging_debug(
"Adding new taxlot to created_tax_lots at index {i}"
.format(i=taxlot_id))
created_tax_lots[taxlot_id] = tl
# Apparently this is how Django clones things?
taxlot_state = original_taxlot_view.state
def unmerge(self, request, pk=None):
"""
Unmerge a taxlot view into two taxlot views
---
parameters:
- name: organization_id
description: The organization_id for this user's organization
required: true
paramType: query
"""
try:
old_view = TaxLotView.objects.select_related(
'taxlot', 'cycle', 'state').get(id=pk,
taxlot__organization_id=self.
request.GET['organization_id'])
except TaxLotView.DoesNotExist:
return {
'status': 'error',
'message': 'taxlot view with id {} does not exist'.format(pk)
}
notes = old_view.notes.all()
for note in notes:
note.taxlot_view = None
merged_state = old_view.state
if merged_state.data_state != DATA_STATE_MATCHING or merged_state.merge_state != MERGE_STATE_MERGED:
return {
'status':
'error',
'message':
'taxlot view with id {} is not a merged taxlot view'.format(pk)
}
log = TaxLotAuditLog.objects.select_related(
'parent_state1', 'parent_state2').filter(
state=merged_state).order_by('-id').first()
if log.parent_state1 is None or log.parent_state2 is None:
return {
'status':
'error',
'message':
'taxlot view with id {} must have two parent states'.format(pk)
}
label = apps.get_model('seed', 'TaxLot_labels')
state1 = log.parent_state1
state2 = log.parent_state2
cycle_id = old_view.cycle_id
# Clone the taxlot record, then the labels
old_taxlot = old_view.taxlot
label_ids = list(old_taxlot.labels.all().values_list('id', flat=True))
new_taxlot = old_taxlot
new_taxlot.id = None
new_taxlot.save()
for label_id in label_ids:
label(taxlot_id=new_taxlot.id, statuslabel_id=label_id).save()
# Create the views
new_view1 = TaxLotView(cycle_id=cycle_id,
taxlot_id=new_taxlot.id,
state=state1)
new_view2 = TaxLotView(cycle_id=cycle_id,
taxlot_id=old_view.taxlot_id,
state=state2)
# Mark the merged state as deleted
merged_state.merge_state = MERGE_STATE_DELETE
merged_state.save()
# Change the merge_state of the individual states
if log.parent1.name in ['Import Creation', 'Manual Edit'
] and log.parent1.import_filename is not None:
# State belongs to a new record
state1.merge_state = MERGE_STATE_NEW
else:
state1.merge_state = MERGE_STATE_MERGED
if log.parent2.name in ['Import Creation', 'Manual Edit'
] and log.parent2.import_filename is not None:
# State belongs to a new record
state2.merge_state = MERGE_STATE_NEW
else:
state2.merge_state = MERGE_STATE_MERGED
state1.save()
state2.save()
# Delete the audit log entry for the merge
log.delete()
# Duplicate pairing
paired_view_ids = list(
TaxLotProperty.objects.filter(taxlot_view_id=old_view.id).order_by(
'property_view_id').values_list('property_view_id', flat=True))
old_view.delete()
new_view1.save()
new_view2.save()
# Duplicate notes to the new views
for note in notes:
created = note.created
updated = note.updated
note.id = None
note.taxlot_view = new_view1
note.save()
ids = [note.id]
note.id = None
note.taxlot_view = new_view2
note.save()
ids.append(note.id)
# Correct the created and updated times to match the original note
Note.objects.filter(id__in=ids).update(created=created,
updated=updated)
for paired_view_id in paired_view_ids:
TaxLotProperty(primary=True,
cycle_id=cycle_id,
taxlot_view_id=new_view1.id,
property_view_id=paired_view_id).save()
TaxLotProperty(primary=True,
cycle_id=cycle_id,
taxlot_view_id=new_view2.id,
property_view_id=paired_view_id).save()
return {'status': 'success', 'view_id': new_view1.id}
def split_taxlots_into_m2m_relationships(self, org_id, org_rules_map):
org = Organization.objects.get(pk=org_id)
logging_info("Splitting tax lot lists for organization {}/{}".format(org_id, org.name))
created_tax_lots = collections.defaultdict(lambda : False)
for m2m in itertools.chain(TaxLotProperty.objects.filter(property_view__property__organization=org).all(),
TaxLotProperty.objects.filter(taxlot_view__taxlot__organization=org).all()):
# aggregate_value_from_state(view.state, org_rules_map[org_id])
# In some cases something in this chain of db calls in m2m.taxlot_view.state.jurisdiction_tax_lot_id
# something is missing. Log it and continue.
try:
jurisdiction_tax_lot_id = m2m.taxlot_view.state.jurisdiction_tax_lot_id
except Exception as e:
logging_error("Error splitting taxlotproperty {t} into m2m: {e}".format(t = m2m, e = e))
continue
logging_info("Starting to do m2m for jurisdiction_tax_lot_id {id}".format(id = jurisdiction_tax_lot_id))
taxlot_id_list = []
try:
taxlot_id_list = get_id_fields(m2m.taxlot_view.state.jurisdiction_tax_lot_id)
logger.info("Found taxlot_id_list: {l}".format(l = taxlot_id_list))
except TaxLotIDValueError, e:
logging_warn(e)
continue
if len(taxlot_id_list) <= 1: continue
logging_info("Tax lot view {} w/ tax_lot id {} was split to {} elements: {}".format(m2m.taxlot_view.pk, m2m.taxlot_view.state.jurisdiction_tax_lot_id,
len(taxlot_id_list), taxlot_id_list))
original_taxlot_view = m2m.taxlot_view
# Some have duplicates
for tax_lot_id in set(taxlot_id_list):
logging_info("Break up tax lot {} to {} for cycle {}".format(tax_lot_id, taxlot_id_list, m2m.cycle))
# Take tax lot and create a taxlot, a taxlot view, and a taxlot state.
# taxlot state, and an m2m for the view and installs each.
# Check to see if the tax lot exists
matching_views_qry = TaxLotView.objects.filter(taxlot__organization=org, state__jurisdiction_tax_lot_id=tax_lot_id)
matching_views_ct = matching_views_qry.count()
logging_info("Found {ct} matching views".format(ct = matching_views_ct))
if matching_views_qry.count():
tax_lot = matching_views_qry.first().taxlot
state = matching_views_qry.first().state
logging_info("Found matching taxlotviews. First is jurisdiction_tax_lot_id {id}".format(id = state.jurisdiction_tax_lot_id))
# FIXME: Yuck! Refactor me please!
created_tax_lots[tax_lot_id] = tax_lot
logging_info("Setting taxlot_state to jurisdiction_tax_lot_id {id}".format(id = original_taxlot_view.state.jurisdiction_tax_lot_id))
# Apparently this is how Django clones things?
taxlot_state = original_taxlot_view.state
taxlot_state.pk = None
taxlot_state.jurisdiction_tax_lot_id = tax_lot_id
logging_info("Setting taxlot_state.jurisdiction_tax_lot_id = {id}".format(id = tax_lot_id))
taxlot_state.save()
else:
logging_info("No match, make a new TaxLot")
tl = TaxLot(organization=m2m.taxlot_view.taxlot.organization)
tl.save()
created_tax_lots[tax_lot_id] = tl
logging_info("Setting taxlot_state to jurisdiction_tax_lot_id {id}".format(id = original_taxlot_view.state.jurisdiction_tax_lot_id))
# Apparently this is how Django clones things?
taxlot_state = original_taxlot_view.state
taxlot_state.pk = None
taxlot_state.jurisdiction_tax_lot_id = tax_lot_id
logging_info("Setting taxlot_state.jurisdiction_tax_lot_id = {id}".format(id = tax_lot_id))
taxlot_state.save()
# Check and see if the Tax Lot View exists
qry = TaxLotView.objects.filter(taxlot = created_tax_lots[tax_lot_id], cycle = m2m.cycle)
taxlotview_ct = qry.count()
logging_info("Found {ct} matching taxlotviews".format(ct = taxlotview_ct))
if taxlotview_ct:
taxlotview = qry.first()
logging_debug("Setting the state of {v} to {s}".format(v = taxlotview.state.jurisdiction_tax_lot_id, s = taxlot_state.jurisdiction_tax_lot_id))
taxlotview.state = taxlot_state
taxlotview.save()
else:
logging_debug("Creating a new TaxLotView with cycle {c} and state {s}".format(c = m2m.cycle.name, s = taxlot_state.jurisdiction_tax_lot_id))
taxlotview = TaxLotView(taxlot = created_tax_lots[tax_lot_id], cycle = m2m.cycle, state = taxlot_state)
# Clone the state from above
taxlotview.save()
logging_debug("TaxLotProperty.objects.get_or_create with pm_id {pm}, jurisdiction_id = {j}, cycle = {c}".format(pm = m2m.property_view.state.pm_property_id, j = taxlotview.state.jurisdiction_tax_lot_id, c = m2m.cycle.name))
TaxLotProperty.objects.get_or_create(property_view = m2m.property_view, taxlot_view = taxlotview, cycle = m2m.cycle)
else:
# The existing TaxLotView and m2m is deleted.
logging_debug("Deleting existing TaxLotView pm {pm}, jurisdiction {j}".format(pm = m2m.property_view.state.pm_property_id, j = m2m.taxlot_view.state.jurisdiction_tax_lot_id))
tl_view = m2m.taxlot_view
m2m.delete()
tl_view.delete()
pass
# Go through each view, find all it's tax lot ids and make sure they don't look like lists of many things.
logging_info("{} => {}".format(jurisdiction_tax_lot_id, taxlot_id_list))
matching_views_qry = TaxLotView.objects.filter(taxlot__organization=org, state__jurisdiction_tax_lot_id=taxlot_id)
matching_views_ct = matching_views_qry.count()
logging_debug("Found {ct} matching views".format(ct = matching_views_ct))
if matching_views_ct:
taxlot = matching_views_qry.first().taxlot
taxlotview_ct = TaxLotView.objects.filter(taxlot = taxlot, cycle = original_taxlot_view.cycle).count()
logging_debug("Found {ct} taxlotviews".format(ct = taxlotview_ct))
if taxlotview_ct == 0:
taxlot_state = original_taxlot_view.state
taxlot_state.pk = None
taxlot_state.jurisdiction_tax_lot_id = taxlot_id
logging_debug("Creating a copy of the original taxlot_view's state with jurisdiction id {j}".format(j = taxlot_id))
taxlot_state.save()
logging_debug("Creating a new TaxLotView")
tlv = TaxLotView(taxlot = taxlot, cycle = original_taxlot_view.cycle, state = taxlot_state)
tlv.save()
else:
logging_debug("Creating a new TaxLot")
tl = TaxLot(organization=original_taxlot_view.taxlot.organization)
tl.save()
logging_debug("Adding new taxlot to created_tax_lots at index {i}".format(i = taxlot_id))
created_tax_lots[taxlot_id] = tl
# Apparently this is how Django clones things?
taxlot_state = original_taxlot_view.state
taxlot_state.pk = None
taxlot_state.jurisdiction_tax_lot_id = taxlot_id
logging_debug("Creating a copy of the original taxlot_view's state with jurisdiction id {j}".format(j = taxlot_id))
taxlot_state.save()