def update_tcms_from_save(self, json_data, save_counter):
# Check save_counter vs queued_save_counters.
queued_save_counter = get_cache_raw(self.QUEUED_TCM_SAVE_COUNTER_KEY, None)
if not queued_save_counter or save_counter > queued_save_counter:
if not get_cache_state(self.UPDATING_TCMS_KEY, None):
set_cache_state(self.UPDATING_TCMS_KEY, True)
for d in json.loads(json_data):
tcm = TableColumnMapping.objects.get(pk=d["pk"])
for field_name in TableColumnMapping.fields_to_save:
if not field_name == "pk":
setattr(tcm, field_name, d[field_name])
tcm.was_a_human_decision = True
tcm.save()
if get_cache_raw(self.QUEUED_TCM_SAVE_COUNTER_KEY, False) is not False:
queued_data = get_cache_raw(self.QUEUED_TCM_DATA_KEY)
queued_time = get_cache_raw(self.QUEUED_TCM_SAVE_COUNTER_KEY)
delete_cache(self.QUEUED_TCM_DATA_KEY)
delete_cache(self.QUEUED_TCM_SAVE_COUNTER_KEY)
delete_cache(self.UPDATING_TCMS_KEY)
self.update_tcms_from_save(queued_data, queued_time)
delete_cache(self.UPDATING_TCMS_KEY)
delete_cache(self.QUEUED_TCM_DATA_KEY)
delete_cache(self.QUEUED_TCM_SAVE_COUNTER_KEY)
return True
else:
set_cache_raw(self.QUEUED_TCM_SAVE_COUNTER_KEY, save_counter)
set_cache_raw(self.QUEUED_TCM_DATA_KEY, json_data)
return False
def get_csv(request):
"""
Download a csv of the results.
"""
import_file_id = request.GET.get('import_file_id')
cleansing_results = get_cache_raw(Cleansing.cache_key(import_file_id))
response = HttpResponse(content_type='text/csv')
response['Content-Disposition'] = 'attachment; filename="Data Cleansing Results.csv"'
writer = csv.writer(response)
writer.writerow(['Address Line 1', 'PM Property ID', 'Tax Lot ID', 'Custom ID', 'Field',
'Error Message', 'Severity'])
for row in cleansing_results:
for result in row['cleansing_results']:
field = result['field']
if field in Cleansing.ASSESSOR_FIELDS_BY_COLUMN:
field = Cleansing.ASSESSOR_FIELDS_BY_COLUMN[field]['title']
writer.writerow([
row['address_line_1'],
row['pm_property_id'],
row['tax_lot_id'],
row['custom_id_1'],
field,
result['message'],
result['severity']
])
return response
def get_cleansing_results(self, request, pk=None):
"""
Retrieve the details of the cleansing script.
---
type:
status:
required: true
type: string
description: either success or error
message:
type: string
description: additional information, if any
progress:
type: integer
description: integer percent of completion
data:
type: JSON
description: object describing the results of the cleansing
parameter_strategy: replace
parameters:
- name: pk
description: Import file ID
required: true
paramType: path
"""
import_file_id = pk
cleansing_results = get_cache_raw(Cleansing.cache_key(import_file_id))
return JsonResponse({
'status': 'success',
'message': 'Cleansing complete',
'progress': 100,
'data': cleansing_results
})
def get_csv(request):
"""
Download a csv of the results.
"""
import_file_id = request.GET.get("import_file_id")
cleansing_results = get_cache_raw(Cleansing.cache_key(import_file_id))
response = HttpResponse(content_type="text/csv")
response["Content-Disposition"] = 'attachment; filename="Data Cleansing Results.csv"'
writer = csv.writer(response)
writer.writerow(
["Address Line 1", "PM Property ID", "Tax Lot ID", "Custom ID", "Field", "Error Message", "Severity"]
)
for row in cleansing_results:
for result in row["cleansing_results"]:
writer.writerow(
[
row["address_line_1"],
row["pm_property_id"],
row["tax_lot_id"],
row["custom_id_1"],
result["formatted_field"],
result["detailed_message"],
result["severity"],
]
)
return response
def get_cleansing_results(request):
"""
Retrieve the details of the cleansing script.
"""
import_file_id = request.GET.get("import_file_id")
cleansing_results = get_cache_raw(Cleansing.cache_key(import_file_id))
return {"status": "success", "message": "Cleansing complete", "progress": 100, "data": cleansing_results}
def get_cleansing_results(request):
"""
Retrieve the details of the cleansing script.
"""
import_file_id = request.GET.get('import_file_id')
cleansing_results = get_cache_raw(Cleansing.cache_key(import_file_id))
# add in additional fields for view
for i, row in enumerate(cleansing_results):
for j, result in enumerate(row['cleansing_results']):
if result['field'] in Cleansing.ASSESSOR_FIELDS_BY_COLUMN:
result['field'] = Cleansing.ASSESSOR_FIELDS_BY_COLUMN[result['field']]['title']
return {
'status': 'success',
'message': 'Cleansing complete',
'progress': 100,
'data': cleansing_results
}
def get_csv(self, request, pk=None):
"""
Download a csv of the results.
---
type:
status:
required: true
type: string
description: either success or error
progress_key:
type: integer
description: ID of background job, for retrieving job progress
parameter_strategy: replace
parameters:
- name: pk
description: Import file ID
required: true
paramType: path
"""
import_file_id = pk
cleansing_results = get_cache_raw(Cleansing.cache_key(import_file_id))
response = HttpResponse(content_type='text/csv')
response['Content-Disposition'] = 'attachment; filename="Data Cleansing Results.csv"'
writer = csv.writer(response)
writer.writerow(['Address Line 1', 'PM Property ID', 'Tax Lot ID', 'Custom ID', 'Field',
'Error Message', 'Severity'])
for row in cleansing_results:
for result in row['cleansing_results']:
writer.writerow([
row['address_line_1'],
row['pm_property_id'],
row['tax_lot_id'],
row['custom_id_1'],
result['formatted_field'],
result['detailed_message'],
result['severity']
])
return response
def save_to_cache(self, file_pk):
"""
Save the results to the cache database. The data in the cache are
stored as a list of dictionaries. The data in this class are stored as
a dict of dict. This is important to remember because the data from the
cache cannot be simply loaded into the above structure.
:param file_pk: Import file primary key
:return: None
"""
# change the format of the data in the cache. Make this a list of
# objects instead of object of objects.
existing_results = get_cache_raw(Cleansing.cache_key(file_pk))
l = []
for key, value in self.results.iteritems():
l.append(value)
existing_results = existing_results + l
z = sorted(existing_results, key=lambda k: k["id"])
set_cache_raw(Cleansing.cache_key(file_pk), z, 3600) # save the results for 1 hour
def premerge_estimated_seconds_remaining(self):
return get_cache_raw(self.premerge_seconds_remaining_key)
def estimated_seconds_remaining(self):
return get_cache_raw(self.merge_seconds_remaining_key)
def test_whole_org_match_merge_link_preview_endpoint_taxlots(self):
# Cycle 1 / ImportFile 1 - Create 1 taxlot
base_taxlot_details = {
'jurisdiction_tax_lot_id': '1st Non-Match Set',
'city': 'City 1',
'district': 'Match Set',
'import_file_id': self.import_file_1.id,
'data_state': DATA_STATE_MAPPING,
'no_default_data': False,
}
tls_1 = self.taxlot_state_factory.get_taxlot_state(
**base_taxlot_details)
self.import_file_1.mapping_done = True
self.import_file_1.save()
geocode_and_match_buildings_task(self.import_file_1.id)
# Cycle 2 / ImportFile 2 - Create 1 unlinked taxlot
base_taxlot_details['jurisdiction_tax_lot_id'] = '2nd Non-Match Set'
base_taxlot_details['district'] = 'Match Set'
base_taxlot_details['import_file_id'] = self.import_file_2.id
tls_2 = self.taxlot_state_factory.get_taxlot_state(
**base_taxlot_details)
self.import_file_2.mapping_done = True
self.import_file_2.save()
geocode_and_match_buildings_task(self.import_file_2.id)
# Check there doesn't exist links
self.assertNotEqual(tls_1.taxlotview_set.first().taxlot_id,
tls_2.taxlotview_set.first().taxlot_id)
url = reverse('api:v3:organizations-match-merge-link-preview',
args=[self.org.id])
post_params = json.dumps({
"inventory_type": "taxlots",
"add": ['district'],
"remove": ['jurisdiction_tax_lot_id']
})
raw_result = self.client.post(url,
post_params,
content_type='application/json')
# Check there *still* doesn't exist links
self.assertNotEqual(tls_1.taxlotview_set.first().taxlot_id,
tls_2.taxlotview_set.first().taxlot_id)
self.assertEqual(200, raw_result.status_code)
raw_content = json.loads(raw_result.content)
identifier = ProgressData.from_key(
raw_content['progress_key']).data['unique_id']
result_key = "org_match_merge_link_result__%s" % identifier
raw_summary = get_cache_raw(result_key)
summary = {str(k): v
for k, v in raw_summary.items() if v} # ignore empty cycles
# Check format of summary
self.assertCountEqual(
[str(self.cycle_1.id), str(self.cycle_2.id)], summary.keys())
# Check that preview shows links would be created
self.assertEqual(summary[str(self.cycle_1.id)][0]['id'],
summary[str(self.cycle_2.id)][0]['id'])
# try to get result using results endpoint
get_result_url = reverse(
'api:v3:organizations-match-merge-link-result',
args=[self.org.id]) + '?match_merge_link_id=' + str(identifier)
get_result_raw_response = self.client.get(get_result_url)
raw_summary = json.loads(get_result_raw_response.content)
summary = {str(k): v
for k, v in raw_summary.items() if v} # ignore empty cycles
# Check format of summary
self.assertCountEqual(
[str(self.cycle_1.id), str(self.cycle_2.id)], summary.keys())
# Check that preview shows links would be created
self.assertEqual(summary[str(self.cycle_1.id)][0]['id'],
summary[str(self.cycle_2.id)][0]['id'])